def prepare_libext_directory():
"""
Creates /usr/hdp/current/oozie/libext-customer and recursively sets
777 permissions on it and its parents.
Also, downloads jdbc driver and provides other staff
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_hdp_stack_version(params.version), '2.2.1.0') >= 0
if not os.path.isdir(params.oozie_libext_customer_dir):
os.makedirs(params.oozie_libext_customer_dir, 0o777)
# ensure that it's rwx for all
os.chmod(params.oozie_libext_customer_dir, 0o777)
# get all hadooplzo* JAR files
# hdp-select set hadoop-client has not run yet, therefore we cannot use
# /usr/hdp/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir))
shutil.copy2(file, params.oozie_libext_customer_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to customer dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir))
shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
def upgrade_oozie_database_and_sharelib(self, env):
"""
Performs the creation and upload of the sharelib and the upgrade of the
database. This method will also perform a kinit if necessary.
It is run before the upgrade of oozie begins exactly once as part of the
upgrade orchestration.
Since this runs before the upgrade has occurred, it should not use any
"current" directories since they will still be pointing to the older
version of Oozie. Instead, it should use versioned directories to ensure
that the commands running are from the oozie version about to be upgraded to.
:return:
"""
import params
env.set_params(params)
Logger.info("Will upgrade the Oozie database")
# get the kerberos token if necessary to execute commands as oozie
if params.security_enabled:
oozie_principal_with_host = params.oozie_principal.replace(
"_HOST", params.hostname)
command = format(
"{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}"
)
Execute(command, user=params.oozie_user, logoutput=True)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# upgrade oozie DB
Logger.info(
format(
'Upgrading the Oozie database, using version {stack_version}'))
# the database upgrade requires the db driver JAR, but since we have
# not yet run <stack-selector-tool> to upgrade the current points, we have to use
# the versioned libext directory as the location[[-vufdtffr,
versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(
stack_version)
oozie.download_database_library_if_needed(
target_directory=versioned_libext_dir)
database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(
stack_version)
Execute(database_upgrade_command,
user=params.oozie_user,
logoutput=True)
# install new sharelib to HDFS
self.create_sharelib(env)
def upgrade_oozie_database_and_sharelib(self, env):
"""
Performs the creation and upload of the sharelib and the upgrade of the
database. This method will also perform a kinit if necessary.
It is run before the upgrade of oozie begins exactly once as part of the
upgrade orchestration.
Since this runs before the upgrade has occurred, it should not use any
"current" directories since they will still be pointing to the older
version of Oozie. Instead, it should use versioned directories to ensure
that the commands running are from the oozie version about to be upgraded to.
:return:
"""
import params
env.set_params(params)
Logger.info("Will upgrade the Oozie database")
# get the kerberos token if necessary to execute commands as oozie
if params.security_enabled:
oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname)
command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}")
Execute(command, user=params.oozie_user, logoutput=True)
upgrade_stack = hdp_select._get_upgrade_stack()
if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
stack_version = upgrade_stack[1]
# upgrade oozie DB
Logger.info(format('Upgrading the Oozie database, using version {stack_version}'))
# the database upgrade requires the db driver JAR, but since we have
# not yet run hdp-select to upgrade the current points, we have to use
# the versioned libext directory as the location[[-vufdtffr,
versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(stack_version)
oozie.download_database_library_if_needed(target_directory=versioned_libext_dir)
database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(stack_version)
Execute(database_upgrade_command, user=params.oozie_user, logoutput=True)
# install new sharelib to HDFS
self.create_sharelib(env)
def upgrade_oozie_database_and_sharelib():
"""
Performs the creation and upload of the sharelib and the upgrade of the
database. This method will also perform a kinit if necessary.
It is run before the upgrade of oozie begins exactly once as part of the
upgrade orchestration.
Since this runs before the upgrade has occurred, it should not use any
"current" directories since they will still be pointing to the older
version of Oozie. Instead, it should use versioned directories to ensure
that the commands running are from the oozie version about to be upgraded to.
:return:
"""
import params
# get the kerberos token if necessary to execute commands as oozie
if params.security_enabled:
oozie_principal_with_host = params.oozie_principal.replace(
"_HOST", params.hostname)
command = format(
"{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}"
)
Execute(command, user=params.oozie_user)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# upgrade oozie DB
Logger.info('Upgrading the Oozie database...')
oozie.download_database_library_if_needed()
database_upgrade_command = "/usr/iop/{0}/oozie/bin/ooziedb.sh upgrade -run".format(
stack_version)
Execute(database_upgrade_command, user=params.oozie_user, logoutput=True)
create_sharelib()
def prepare_libext_directory():
"""
Performs the following actions on libext:
- creates /usr/hdp/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_hdp_stack_version(params.version), '2.2.1.0') >= 0
# ensure the directory exists
Directory(params.oozie_libext_dir, mode = 0777)
# get all hadooplzo* JAR files
# hdp-select set hadoop-client has not run yet, therefore we cannot use
# /usr/hdp/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to libext dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
# something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
File(oozie_ext_zip_target_path,
mode=0644
)
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = hdp_select._get_upgrade_stack()
if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
# something like 2.3.0.0-1234
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version)
Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'))
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def prepare_libext_directory():
"""
Performs the following actions on libext:
- creates /usr/hdp/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_hdp_stack_version(params.version), '2.2.1.0') >= 0
# ensure the directory exists
Directory(params.oozie_libext_dir, mode = 0777)
# get all hadooplzo* JAR files
# hdp-select set hadoop-client has not run yet, therefore we cannot use
# /usr/hdp/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to libext dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
# something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
sudo.chmod(oozie_ext_zip_target_path, 0644)
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = hdp_select._get_upgrade_stack()
if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
# something like 2.3.0.0-1234
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version)
Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'))
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def prepare_libext_directory(upgrade_type=None):
"""
Performs the following actions on libext:
- creates <stack-root>/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some stack versions don't need the lzo compression libraries
target_version_needs_compression_libraries = params.version and check_stack_feature(
StackFeature.LZO, params.version)
# ensure the directory exists
Directory(params.oozie_libext_dir, mode=0777)
# get all hadooplzo* JAR files
# <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
# <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present.
#
# This can also be called during a Downgrade.
#
# When a version is Installed, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
#
# This block is just copying around files - there is no assumption about installation
#
if params.lzo_enabled and (
params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format(
"{stack_root}/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir,
params.ext_js_file)
# Copy ext ZIP to libext dir
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(
upgrade_type, params)
found_at_least_one_oozie_ext_file = False
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
found_at_least_one_oozie_ext_file = True
Logger.info("Copying {0} to {1}".format(
source_ext_zip_path, params.oozie_libext_dir))
Execute(
("cp", source_ext_zip_path, params.oozie_libext_dir),
sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"),
oozie_ext_zip_target_path),
sudo=True)
File(oozie_ext_zip_target_path, mode=0644)
break
if not found_at_least_one_oozie_ext_file:
raise Fail(
"Unable to find any Oozie source extension files from the following paths {0}"
.format(source_ext_zip_paths))
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(
params.stack_root, stack_version)
Logger.info("Copying {0} to {1}".format(
versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(
format(
'{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'
))
Execute(
format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
))
