def expand_mpack(archive_path):
"""
Expand management pack
:param archive_path: Local path to management pack
:return: Path where the management pack was expanded
"""
tmpdir = _get_temp_dir()
archive_root_dir = get_archive_root_dir(archive_path)
if not archive_root_dir:
print_error_msg("Malformed management pack. Root directory missing!")
raise FatalException(-1, 'Malformed management pack. Root directory missing!')
# Expand management pack in temp directory
tmp_root_dir = os.path.join(tmpdir, archive_root_dir)
print_info_msg("Expand management pack at temp location {0}".format(tmp_root_dir))
if os.path.exists(tmp_root_dir):
sudo.rmtree(tmp_root_dir)
with Environment():
untar_archive(archive_path, tmpdir)
if not os.path.exists(tmp_root_dir):
print_error_msg("Malformed management pack. Failed to expand management pack!")
raise FatalException(-1, 'Malformed management pack. Failed to expand management pack!')
return tmp_root_dir
def restore_configuration():
"""
Restores the configuration backups to their proper locations after an
upgrade has completed.
:return:
"""
Logger.info('Restoring Oozie configuration directory after upgrade...')
directoryMappings = OozieUpgrade._get_directory_mappings()
for directory in directoryMappings:
archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
directoryMappings[directory])
if not os.path.isfile(archive):
raise Fail(
"Unable to restore missing backup archive {0}".format(
archive))
Logger.info('Extracting {0} to {1}'.format(archive, directory))
tar_archive.untar_archive(archive, directory)
# cleanup
Directory(
os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR),
action="delete",
)
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
absolute_backup_dir = None
if params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE:
Logger.info("Backing up directories. Initial conf folder: %s" % os.path.realpath(params.knox_conf_dir))
# This will backup the contents of the conf directory into /tmp/knox-upgrade-backup/knox-conf-backup.tar
absolute_backup_dir = upgrade.backup_data()
# conf-select will change the symlink to the conf folder.
conf_select.select(params.stack_name, "knox", params.version)
hdp_select.select("knox-server", params.version)
# Extract the tar of the old conf folder into the new conf directory
if absolute_backup_dir is not None and params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE:
conf_tar_source_path = os.path.join(absolute_backup_dir, upgrade.BACKUP_CONF_ARCHIVE)
if os.path.exists(conf_tar_source_path):
extract_dir = os.path.realpath(params.knox_conf_dir)
conf_tar_dest_path = os.path.join(extract_dir, upgrade.BACKUP_CONF_ARCHIVE)
Logger.info("Copying %s into %s file." % (upgrade.BACKUP_CONF_ARCHIVE, conf_tar_dest_path))
Execute(('cp', conf_tar_source_path, conf_tar_dest_path),
sudo = True,
)
tar_archive.untar_archive(conf_tar_source_path, extract_dir)
File(conf_tar_dest_path,
action = "delete",
)
def pre_rolling_restart(self, env):
import params
env.set_params(params)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
absolute_backup_dir = None
if params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE:
Logger.info("Backing up directories. Initial conf folder: %s" % os.path.realpath(params.knox_conf_dir))
# This will backup the contents of the conf directory into /tmp/knox-upgrade-backup/knox-conf-backup.tar
absolute_backup_dir = upgrade.backup_data()
# conf-select will change the symlink to the conf folder.
conf_select.select(params.stack_name, "knox", params.version)
hdp_select.select("knox-server", params.version)
# Extract the tar of the old conf folder into the new conf directory
if absolute_backup_dir is not None and params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE:
conf_tar_source_path = os.path.join(absolute_backup_dir, upgrade.BACKUP_CONF_ARCHIVE)
if os.path.exists(conf_tar_source_path):
extract_dir = os.path.realpath(params.knox_conf_dir)
conf_tar_dest_path = os.path.join(extract_dir, upgrade.BACKUP_CONF_ARCHIVE)
Logger.info("Copying %s into %s file." % (upgrade.BACKUP_CONF_ARCHIVE, conf_tar_dest_path))
Execute(('cp', conf_tar_source_path, conf_tar_dest_path),
sudo = True,
)
tar_archive.untar_archive(conf_tar_source_path, extract_dir)
File(conf_tar_dest_path,
action = "delete",
)
def pre_start_restore():
"""
Restores the flume config, config dir, file/spillable channels to their proper locations
after an upgrade has completed.
:return:
"""
Logger.info('Restoring Flume data and configuration after upgrade...')
directoryMappings = _get_directory_mappings()
for directory in directoryMappings:
archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
directoryMappings[directory])
if os.path.isfile(archive):
tar_archive.untar_archive(archive, directory)
# cleanup
if os.path.exists(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)):
shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
def pre_start_restore():
"""
Restores the directory backups to their proper locations
after an upgrade has completed.
:return:
"""
Logger.info('Restoring Beacon backed up directories after upgrade...')
directoryMappings = _get_directory_mappings()
for directory in directoryMappings:
archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
directoryMappings[directory])
if not os.path.isfile(archive):
raise Fail("Unable to restore missing backup archive {0}".format(archive))
tar_archive.untar_archive(archive, directory)
# cleanup
Directory(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR), action = "delete" )
def pre_start_restore():
"""
Restores the data and configuration backups to their proper locations
after an upgrade has completed.
:return:
"""
Logger.info('Restoring Falcon data and configuration directories after upgrade...')
directoryMappings = _get_directory_mappings()
for directory in directoryMappings:
archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
directoryMappings[directory])
if not os.path.isfile(archive):
raise Fail("Unable to restore missing backup archive {0}".format(archive))
tar_archive.untar_archive(archive, directory)
# cleanup
Directory(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR),
action = "delete",
)
def _prepare_tez_tarball():
"""
Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
It's very important to use the version of mapreduce which matches tez here.
Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the
GPL license has been accepted.
:return: the full path of the newly created tez tarball to use
"""
import tempfile
Logger.info("Preparing the Tez tarball...")
# get the mapreduce tarball which matches the version of tez
# tez installs the mapreduce tar, so it should always be present
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
_, tez_source_file, _, _ = get_tarball_paths("tez")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
tez_temp_dir = tempfile.mkdtemp(prefix="tez-tarball-", dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
sudo.chmod(tez_temp_dir, 0777)
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir))
tar_archive.untar_archive(tez_source_file, tez_temp_dir)
hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib",
"native")
tez_lib_dir = os.path.join(tez_temp_dir, "lib")
if not os.path.exists(hadoop_lib_native_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_dir))
if not os.path.exists(tez_lib_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist"
.format(tez_lib_dir))
# copy native libraries from hadoop to tez
Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo=True)
# if enabled, LZO GPL libraries must be copied as well
if lzo_utils.should_install_lzo():
stack_root = Script.get_stack_root()
service_version = component_version.get_component_repository_version(
service_name="TEZ")
# some installations might not have Tez, but MapReduce2 should be a fallback to get the LZO libraries from
if service_version is None:
Logger.warning(
"Tez does not appear to be installed, using the MapReduce version to get the LZO libraries"
)
service_version = component_version.get_component_repository_version(
service_name="MAPREDUCE2")
hadoop_lib_native_lzo_dir = os.path.join(stack_root, service_version,
"hadoop", "lib", "native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
Logger.warning(
"Unable to located native LZO libraries at {0}, falling back to hadoop home"
.format(hadoop_lib_native_lzo_dir))
hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current",
"hadoop-client", "lib",
"native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}"
.format(hadoop_lib_native_lzo_dir))
Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir),
sudo=True)
# ensure that the tez/lib directory is readable by non-root (which it typically is not)
Directory(tez_lib_dir, mode=0755, cd_access='a', recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
tez_native_tarball_staging_dir = os.path.join(
temp_dir, "tez-native-tarball-staging")
if not os.path.exists(tez_native_tarball_staging_dir):
Directory(tez_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
tez_tarball_with_native_lib = os.path.join(tez_native_tarball_staging_dir,
"tez-native.tar.gz")
Logger.info("Creating a new Tez tarball at {0}".format(
tez_tarball_with_native_lib))
# tar up Tez, making sure to specify nothing for the arcname so that it does not include an absolute path
with closing(tarfile.open(tez_tarball_with_native_lib,
"w:gz")) as new_tez_tarball:
new_tez_tarball.add(tez_temp_dir, arcname=os.path.sep)
# ensure that the tarball can be read and uploaded
sudo.chmod(tez_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
sudo.rmtree(tez_temp_dir)
return tez_tarball_with_native_lib
def _prepare_mapreduce_tarball():
"""
Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO is
not enabled or has not been opted-in, then this will do nothing and return the original
tarball to upload to HDFS.
:return: the full path of the newly created mapreduce tarball to use or the original path
if no changes were made
"""
# get the mapreduce tarball to crack open and add LZO libraries to
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
if not lzo_utils.should_install_lzo():
return mapreduce_source_file
Logger.info("Preparing the mapreduce tarball with native LZO libraries...")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
# calculate the source directory for LZO
hadoop_lib_native_source_dir = os.path.join(
os.path.dirname(mapreduce_source_file), "lib", "native")
if not sudo.path_exists(hadoop_lib_native_source_dir):
raise Fail(
"Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_source_dir))
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir)
mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib")
# copy native libraries from source hadoop to target
Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir),
sudo=True)
# ensure that the hadoop/lib/native directory is readable by non-root (which it typically is not)
Directory(mapreduce_lib_dir,
mode=0755,
cd_access='a',
recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
mapreduce_native_tarball_staging_dir = os.path.join(
temp_dir, "mapreduce-native-tarball-staging")
if not os.path.exists(mapreduce_native_tarball_staging_dir):
Directory(mapreduce_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
mapreduce_tarball_with_native_lib = os.path.join(
mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz")
Logger.info("Creating a new mapreduce tarball at {0}".format(
mapreduce_tarball_with_native_lib))
tar_archive.archive_dir_via_temp_file(mapreduce_tarball_with_native_lib,
mapreduce_temp_dir)
# ensure that the tarball can be read and uploaded
sudo.chmod(mapreduce_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
return mapreduce_tarball_with_native_lib
