def configure_atlas_user_for_tagsync(self, env):
Logger.info("Configuring Atlas user for Tagsync service.")
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_name = upgrade_stack[0]
stack_version = upgrade_stack[1]
stack_select.select("ranger-tagsync", stack_version)
conf_select.select(stack_name, "ranger-tagsync", stack_version)
if params.stack_supports_ranger_tagsync_ssl_xml_support:
Logger.info(
"Upgrading Tagsync, stack support Atlas user for Tagsync, creating keystore for same."
)
self.create_atlas_user_keystore(env)
else:
Logger.info(
"Upgrading Tagsync, stack does not support Atlas user for Tagsync, skipping keystore creation for same."
)
Logger.info("Configuring Atlas user for Tagsync service done.")
def upgrade_oozie_database_and_sharelib(self, env):
"""
Performs the creation and upload of the sharelib and the upgrade of the
database. This method will also perform a kinit if necessary.
It is run before the upgrade of oozie begins exactly once as part of the
upgrade orchestration.
Since this runs before the upgrade has occurred, it should not use any
"current" directories since they will still be pointing to the older
version of Oozie. Instead, it should use versioned directories to ensure
that the commands running are from the oozie version about to be upgraded to.
:return:
"""
import params
env.set_params(params)
Logger.info("Will upgrade the Oozie database")
# get the kerberos token if necessary to execute commands as oozie
if params.security_enabled:
oozie_principal_with_host = params.oozie_principal.replace(
"_HOST", params.hostname)
command = format(
"{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}"
)
Execute(command, user=params.oozie_user, logoutput=True)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# upgrade oozie DB
Logger.info(
format(
'Upgrading the Oozie database, using version {stack_version}'))
# the database upgrade requires the db driver JAR, but since we have
# not yet run <stack-selector-tool> to upgrade the current points, we have to use
# the versioned libext directory as the location[[-vufdtffr,
versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(
stack_version)
oozie.download_database_library_if_needed(
target_directory=versioned_libext_dir)
database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(
stack_version)
Execute(database_upgrade_command,
user=params.oozie_user,
logoutput=True)
# install new sharelib to HDFS
self.create_sharelib(env)
def set_pre_start(self, env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_select.select_packages(params.version)
def setup_ranger_kms_database(self, env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_version = upgrade_stack[1]
Logger.info(format('Setting Ranger KMS database schema, using version {stack_version}'))
setup_kms_db(stack_version=stack_version)
def set_pre_start(self, env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_name = upgrade_stack[0]
stack_version = upgrade_stack[1]
stack_select.select("ranger-admin", stack_version)
conf_select.select(stack_name, "ranger-admin", stack_version)
def setup_ranger_java_patches(self, env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_version = upgrade_stack[1]
if params.xml_configurations_supported:
Logger.info(format('Applying Ranger java patches, using version {stack_version}'))
from setup_ranger_xml import setup_java_patch
setup_java_patch(stack_version=stack_version)
def setup_ranger_database(self, env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_version = upgrade_stack[1]
if params.xml_configurations_supported:
Logger.info(format('Setting Ranger database schema, using version {stack_version}'))
from setup_ranger_xml import setup_ranger_db
setup_ranger_db(stack_version=stack_version)
def setup_ranger_java_patches(self, env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_version = upgrade_stack[1]
if params.upgrade_direction == Direction.UPGRADE:
target_version = upgrade_summary.get_target_version(
"RANGER", default_version=stack_version)
Logger.info(
format(
'Applying Ranger java patches, using version {target_version}'
))
setup_ranger_xml.setup_java_patch(stack_version=target_version)
def create_sharelib(self, env):
"""
Performs the creation and upload of the sharelib.
This method will also perform a kinit if necessary.
It is run before the upgrade of oozie begins exactly once as part of the
upgrade orchestration.
Since this runs before the upgrade has occurred, it should not use any
"current" directories since they will still be pointing to the older
version of Oozie. Instead, it should use versioned directories to ensure
that the commands running are from the oozie version about to be upgraded to.
:param env:
:return:
"""
import params
env.set_params(params)
Logger.info('Creating a new sharelib and uploading it to HDFS...')
# ensure the oozie directory exists for the sharelib
params.HdfsResource(format("{oozie_hdfs_user_dir}/share"),
action="create_on_execute",
type="directory",
owner="oozie",
group="hadoop",
mode=0755,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# install new sharelib to HDFS
sharelib_command = "/usr/hdp/{0}/oozie/bin/oozie-setup.sh sharelib create -fs {1}".format(
stack_version, params.fs_root)
Execute(sharelib_command, user=params.oozie_user, logoutput=True)
def setup_tls_toolkit_upgrade(self,env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_config_dir + '/config_version'
client_json_file = params.nifi_config_dir+ '/nifi-certificate-authority-client.json'
if sudo.path_isfile(version_file):
Logger.info(format('Remove config version file if does exist'))
sudo.unlink(version_file)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def setup_ranger_database(self, env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
stack_version = upgrade_stack[1]
if params.xml_configurations_supported and params.upgrade_direction == Direction.UPGRADE:
target_version = upgrade_summary.get_target_version(
"RANGER", default_version=stack_version)
Logger.info(
format(
'Setting Ranger database schema, using version {target_version}'
))
from setup_ranger_xml import setup_ranger_db
setup_ranger_db(stack_version=target_version)
def setup_tls_toolkit_upgrade(self,env):
import params
env.set_params(params)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None:
raise Fail('Unable to determine the stack and stack version')
if params.upgrade_direction == Direction.UPGRADE and params.nifi_registry_ssl_enabled and params.nifi_ca_host:
version_file = params.nifi_registry_config_dir + '/config_version'
client_json_file = params.nifi_registry_config_dir+ '/nifi-certificate-authority-client.json'
# if not sudo.path_isfile(version_file):
# Logger.info(format('Create config version file if it does not exist'))
# version_num = params.config['configurationTags']['nifi-registry-ambari-ssl-config']['tag']
# nifi_toolkit_util.save_config_version(version_file,'ssl',version_num,params.nifi_registry_user,params.nifi_registry_group)
if sudo.path_isfile(client_json_file):
Logger.info(format('Remove client json file'))
sudo.unlink(client_json_file)
def upgrade_oozie_database_and_sharelib():
"""
Performs the creation and upload of the sharelib and the upgrade of the
database. This method will also perform a kinit if necessary.
It is run before the upgrade of oozie begins exactly once as part of the
upgrade orchestration.
Since this runs before the upgrade has occurred, it should not use any
"current" directories since they will still be pointing to the older
version of Oozie. Instead, it should use versioned directories to ensure
that the commands running are from the oozie version about to be upgraded to.
:return:
"""
import params
# get the kerberos token if necessary to execute commands as oozie
if params.security_enabled:
oozie_principal_with_host = params.oozie_principal.replace(
"_HOST", params.hostname)
command = format(
"{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}"
)
Execute(command, user=params.oozie_user)
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# upgrade oozie DB
Logger.info('Upgrading the Oozie database...')
oozie.download_database_library_if_needed()
database_upgrade_command = "/usr/iop/{0}/oozie/bin/ooziedb.sh upgrade -run".format(
stack_version)
Execute(database_upgrade_command, user=params.oozie_user, logoutput=True)
create_sharelib()
stack_version_formatted = status_params.stack_version_formatted
major_stack_version = get_major_version(stack_version_formatted)
version_for_stack_feature_checks = get_stack_feature_version(config)
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
hadoop_lib_home = stack_select.get_hadoop_dir("lib")
#spark_conf
spark_conf_dir = format("{stack_root}/current/spark-client/conf")
#hadoop params
if stack_version_formatted and check_stack_feature(
StackFeature.ROLLING_UPGRADE, stack_version_formatted):
stack_version = None
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is not None and len(
upgrade_stack) == 2 and upgrade_stack[1] is not None:
stack_version = upgrade_stack[1]
# oozie-server or oozie-client, depending on role
oozie_root = status_params.component_directory
# using the correct oozie root dir, format the correct location
oozie_lib_dir = format("{stack_root}/current/{oozie_root}")
oozie_setup_sh = format(
"{stack_root}/current/{oozie_root}/bin/oozie-setup.sh")
oozie_webapps_dir = format(
"{stack_root}/current/{oozie_root}/oozie-server/webapps")
oozie_webapps_conf_dir = format(
"{stack_root}/current/{oozie_root}/oozie-server/conf")
def prepare_libext_directory():
"""
Performs the following actions on libext:
- creates <stack-root>/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_stack_version(params.version), '2.2.1.0') >= 0
# ensure the directory exists
Directory(params.oozie_libext_dir, mode=0777)
# get all hadooplzo* JAR files
# <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
# <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (
params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to libext dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir,
"ext-2.2.zip")
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(
oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file,
params.oozie_libext_dir))
Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"),
oozie_ext_zip_target_path),
sudo=True)
File(oozie_ext_zip_target_path, mode=0644)
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
# something like 2.3.0.0-1234
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(
stack_version)
Logger.info("Copying {0} to {1}".format(
versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(
format(
'{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'
))
Execute(
format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
))
def prepare_libext_directory(upgrade_type=None):
"""
Performs the following actions on libext:
- creates <stack-root>/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some stack versions don't need the lzo compression libraries
target_version_needs_compression_libraries = params.version and check_stack_feature(
StackFeature.LZO, params.version)
# ensure the directory exists
Directory(params.oozie_libext_dir, mode=0777)
# get all hadooplzo* JAR files
# <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use
# <stack-root>/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present.
#
# This can also be called during a Downgrade.
#
# When a version is Installed, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
#
# This block is just copying around files - there is no assumption about installation
#
if params.lzo_enabled and (
params.upgrade_direction == Direction.UPGRADE
or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format(
"{stack_root}/{version}/hadoop/lib")
files = glob.iglob(
os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(
str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# something like <stack-root>/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir,
params.ext_js_file)
# Copy ext ZIP to libext dir
# Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths(
upgrade_type, params)
found_at_least_one_oozie_ext_file = False
# Copy the first oozie ext-2.2.zip file that is found.
# This uses a list to handle the cases when migrating from some versions of BigInsights to HDP.
if source_ext_zip_paths is not None:
for source_ext_zip_path in source_ext_zip_paths:
if os.path.isfile(source_ext_zip_path):
found_at_least_one_oozie_ext_file = True
Logger.info("Copying {0} to {1}".format(
source_ext_zip_path, params.oozie_libext_dir))
Execute(
("cp", source_ext_zip_path, params.oozie_libext_dir),
sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"),
oozie_ext_zip_target_path),
sudo=True)
File(oozie_ext_zip_target_path, mode=0644)
break
if not found_at_least_one_oozie_ext_file:
raise Fail(
"Unable to find any Oozie source extension files from the following paths {0}"
.format(source_ext_zip_paths))
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = stack_select._get_upgrade_stack()
if upgrade_stack is None or len(
upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail(
"Unable to determine the stack that is being upgraded to or downgraded to."
)
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(
params.stack_root, stack_version)
Logger.info("Copying {0} to {1}".format(
versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(
format(
'{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'
))
Execute(
format(
'{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'
))