def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s since it doesn't exist." % self.resource)
# dead links should be followed, else we gonna have failures on trying to create directories on top of them.
if self.resource.follow:
# Follow symlink until the tail.
followed_links = set()
while sudo.path_lexists(path):
if path in followed_links:
raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path))
followed_links.add(path)
path = sudo.readlink(path)
if path != self.resource.path:
Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path))
if self.resource.create_parents:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
_ensure_metadata(path, self.resource.owner, self.resource.group,
mode=self.resource.mode, cd_access=self.resource.cd_access,
recursive_ownership=self.resource.recursive_ownership, recursive_mode_flags=self.resource.recursive_mode_flags,
recursion_follow_links=self.resource.recursion_follow_links, safemode_folders=self.resource.safemode_folders)
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s" % self.resource)
if self.resource.recursive:
if self.resource.recursive_permission:
DirectoryProvider.makedirs_and_set_permission_recursively(
path, self.resource.owner, self.resource.group,
self.resource.mode)
else:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail(
"Applying %s failed, parent directory %s doesn't exist"
% (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" %
(self.resource, path))
_ensure_metadata(path,
self.resource.owner,
self.resource.group,
mode=self.resource.mode,
cd_access=self.resource.cd_access)
def action_create(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
write = False
content = self._get_content()
if not sudo.path_exists(path):
write = True
reason = "it doesn't exist"
elif self.resource.replace:
if content is not None:
old_content = sudo.read_file(path, encoding=self.resource.encoding)
if content != old_content:
write = True
reason = "contents don't match"
if self.resource.backup:
self.resource.env.backup_file(path)
if write:
Logger.info("Writing %s because %s" % (self.resource, reason))
sudo.create_file(path, content, encoding=self.resource.encoding)
_ensure_metadata(self.resource.path, self.resource.owner,
self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def action_create(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
write = False
content = self._get_content()
if not sudo.path_exists(path):
write = True
reason = "it doesn't exist"
elif self.resource.replace:
if content is not None:
old_content = sudo.read_file(path, encoding=self.resource.encoding)
if content != old_content:
write = True
reason = "contents don't match"
if self.resource.backup:
self.resource.env.backup_file(path)
if write:
Logger.info("Writing %s because %s" % (self.resource, reason))
sudo.create_file(path, content, encoding=self.resource.encoding)
_ensure_metadata(self.resource.path, self.resource.owner,
self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def copytree(src, dst, exclude_sub_dirs=set(), force_replace=False):
"""
Copy content of directory from source path to the target path with possibility to exclude some directories
:type src str
:type dst str
:type exclude_sub_dirs list|set
:type force_replace bool
"""
def copy(_src, _dst):
from resource_management.core import shell
# full path is required to avoid usage of system command aliases like 'cp -i', which block overwrite
if force_replace:
shell.checked_call(["/bin/cp", "-rfp", _src, _dst], sudo=True)
else:
shell.checked_call(["/bin/cp", "-rp", _src, _dst], sudo=True)
if not sudo.path_isdir(src) or not sudo.path_isdir(dst):
raise Fail("The source or the destination is not a folder")
sub_dirs_to_copy = sudo.listdir(src)
for d in sub_dirs_to_copy:
if d not in exclude_sub_dirs:
src_path = os.path.join(src, d)
copy(src_path, dst)
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s since it doesn't exist." %
self.resource)
# dead links should be followed, else we gonna have failures on trying to create directories on top of them.
if self.resource.follow:
# Follow symlink until the tail.
followed_links = set()
while sudo.path_islink(path):
if path in followed_links:
raise Fail(
"Applying %s failed, looped symbolic links found while resolving %s"
% (self.resource, path))
followed_links.add(path)
prev_path = path
path = sudo.readlink(path)
if not os.path.isabs(path):
path = os.path.join(os.path.dirname(prev_path), path)
if path != self.resource.path:
Logger.info(
"Following the link {0} to {1} to create the directory"
.format(self.resource.path, path))
if self.resource.create_parents:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail(
"Applying %s failed, parent directory %s doesn't exist"
% (self.resource, dirname))
try:
sudo.makedir(path, self.resource.mode or 0755)
except Exception as ex:
# race condition (somebody created the file before us)
if "File exists" in str(ex):
sudo.makedirs(path, self.resource.mode or 0755)
else:
raise
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" %
(self.resource, path))
_ensure_metadata(
path,
self.resource.owner,
self.resource.group,
mode=self.resource.mode,
cd_access=self.resource.cd_access,
recursive_ownership=self.resource.recursive_ownership,
recursive_mode_flags=self.resource.recursive_mode_flags,
recursion_follow_links=self.resource.recursion_follow_links,
safemode_folders=self.resource.safemode_folders)
def get_hadoop_dir(target):
"""
Return the hadoop shared directory which should be used for the command's component. The
directory including the component's version is tried first, but if that doesn't exist,
this will fallback to using "current".
:target: the target directory
"""
stack_root = Script.get_stack_root()
stack_version = Script.get_stack_version()
if not target in HADOOP_DIR_DEFAULTS:
raise Fail("Target {0} not defined".format(target))
hadoop_dir = HADOOP_DIR_DEFAULTS[target]
formatted_stack_version = format_stack_version(stack_version)
if stack_features.check_stack_feature(StackFeature.ROLLING_UPGRADE, formatted_stack_version):
# read the desired version from the component map and use that for building the hadoop home
version = component_version.get_component_repository_version()
if version is None:
version = default("/commandParams/version", None)
# home uses a different template
if target == "home":
hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, version, "hadoop")
if version is None or sudo.path_isdir(hadoop_dir) is False:
hadoop_dir = HADOOP_HOME_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client")
else:
hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, version, "hadoop", target)
if version is None or sudo.path_isdir(hadoop_dir) is False:
hadoop_dir = HADOOP_DIR_TEMPLATE.format(stack_root, "current", "hadoop-client", target)
return hadoop_dir
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s since it doesn't exist." % self.resource)
# dead links should be followed, else we gonna have failures on trying to create directories on top of them.
if self.resource.follow:
# Follow symlink until the tail.
followed_links = set()
while sudo.path_lexists(path):
if path in followed_links:
raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path))
followed_links.add(path)
path = sudo.readlink(path)
if path != self.resource.path:
Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path))
if self.resource.recursive:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
_ensure_metadata(path, self.resource.owner, self.resource.group,
mode=self.resource.mode, cd_access=self.resource.cd_access)
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
stat = sudo.stat(path)
if user:
uid = _coerce_uid(user)
if stat.st_uid != uid:
Logger.info("Changing owner for %s from %d to %s" %
(path, stat.st_uid, user))
sudo.chown(path, user, None)
if group:
gid = _coerce_gid(group)
if stat.st_gid != gid:
Logger.info("Changing group for %s from %d to %s" %
(path, stat.st_gid, group))
sudo.chown(path, None, group)
if mode:
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" %
(path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access + "+x")
dir_path = os.path.split(dir_path)[0]
def action_create(self):
path = self.resource.path
if sudo.path_lexists(path):
oldpath = os.path.realpath(path)
if oldpath == self.resource.to:
return
if not sudo.path_lexists(path):
raise Fail(
"%s trying to create a symlink with the same name as an existing file or directory"
% self.resource)
Logger.info("%s replacing old symlink to %s" %
(self.resource, oldpath))
sudo.unlink(path)
if self.resource.hard:
if not sudo.path_exists(self.resource.to):
raise Fail(
"Failed to apply %s, linking to nonexistent location %s" %
(self.resource, self.resource.to))
if sudo.path_isdir(self.resource.to):
raise Fail(
"Failed to apply %s, cannot create hard link to a directory (%s)"
% (self.resource, self.resource.to))
Logger.info("Creating hard %s" % self.resource)
sudo.link(self.resource.to, path)
else:
if not sudo.path_exists(self.resource.to):
Logger.info("Warning: linking to nonexistent location %s" %
self.resource.to)
Logger.info("Creating symbolic %s to %s" %
(self.resource, self.resource.to))
sudo.symlink(self.resource.to, path)
def copy_toolkit_scripts(toolkit_files_dir, toolkit_tmp_dir, user, group,
upgrade_type):
nifiToolkitDirFilesPath = None
nifiToolkitDirTmpPath = None
Logger.info("Toolkit files dir is " + toolkit_files_dir)
Logger.info("Toolkit tmp dir is " + toolkit_tmp_dir)
for dir in os.listdir(toolkit_files_dir):
if dir.startswith('nifi-toolkit-'):
nifiToolkitDirFilesPath = os.path.join(toolkit_files_dir, dir)
nifiToolkitDirTmpPath = os.path.join(toolkit_tmp_dir, dir)
if not sudo.path_isdir(nifiToolkitDirTmpPath) or not (upgrade_type is
None):
os.system("\cp -r " + nifiToolkitDirFilesPath + " " + toolkit_tmp_dir)
Directory(nifiToolkitDirTmpPath,
owner=user,
group=group,
create_parents=False,
recursive_ownership=True,
cd_access="a",
mode=0755)
os.system("\/var/lib/ambari-agent/ambari-sudo.sh chmod -R 755 " +
nifiToolkitDirTmpPath)
def get_hadoop_conf_dir():
"""
Return the hadoop shared conf directory which should be used for the command's component. The
directory including the component's version is tried first, but if that doesn't exist,
this will fallback to using "current".
"""
stack_root = Script.get_stack_root()
stack_version = Script.get_stack_version()
hadoop_conf_dir = os.path.join(os.path.sep, "etc", "hadoop", "conf")
if check_stack_feature(StackFeature.CONFIG_VERSIONING, stack_version):
# read the desired version from the component map and use that for building the hadoop home
version = component_version.get_component_repository_version()
if version is None:
version = default("/commandParams/version", None)
hadoop_conf_dir = os.path.join(stack_root, str(version), "hadoop",
"conf")
if version is None or sudo.path_isdir(hadoop_conf_dir) is False:
hadoop_conf_dir = os.path.join(stack_root, "current",
"hadoop-client", "conf")
Logger.info("Using hadoop conf dir: {0}".format(hadoop_conf_dir))
return hadoop_conf_dir
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
stat = sudo.stat(path)
if user:
uid = _coerce_uid(user)
if stat.st_uid != uid:
Logger.info(
"Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
sudo.chown(path, user, None)
if group:
gid = _coerce_gid(group)
if stat.st_gid != gid:
Logger.info(
"Changing group for %s from %d to %s" % (path, stat.st_gid, group))
sudo.chown(path, None, group)
if mode:
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" % (
path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access+"+x")
dir_path = os.path.split(dir_path)[0]
def action_create(self):
path = self.resource.path
if sudo.path_lexists(path):
oldpath = os.path.realpath(path)
if oldpath == self.resource.to:
return
if not sudo.path_lexists(path):
raise Fail(
"%s trying to create a symlink with the same name as an existing file or directory" % self.resource)
Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath))
sudo.unlink(path)
if self.resource.hard:
if not sudo.path_exists(self.resource.to):
raise Fail("Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to))
if sudo.path_isdir(self.resource.to):
raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to))
Logger.info("Creating hard %s" % self.resource)
sudo.link(self.resource.to, path)
else:
if not sudo.path_exists(self.resource.to):
Logger.info("Warning: linking to nonexistent location %s" % self.resource.to)
Logger.info("Creating symbolic %s to %s" % (self.resource, self.resource.to))
sudo.symlink(self.resource.to, path)
def action_delete(self):
path = self.resource.path
if sudo.path_exists(path):
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
Logger.info("Removing directory %s and all its content" % self.resource)
sudo.rmtree(path)
def action_delete(self):
path = self.resource.path
if sudo.path_exists(path):
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
Logger.info("Removing directory %s and all its content" % self.resource)
sudo.rmtree(path)
def action_delete(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def action_delete(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def _copy_from_local_directory(self, target, source):
for next_path_part in sudo.listdir(source):
new_source = os.path.join(source, next_path_part)
new_target = format("{target}/{next_path_part}")
if sudo.path_isdir(new_source):
Logger.info(format("Creating DFS directory {new_target}"))
self._create_directory(new_target)
self._copy_from_local_directory(new_target, new_source)
else:
self._create_file(new_target, new_source)
def _copy_from_local_directory(self, target, source):
for next_path_part in sudo.listdir(source):
new_source = os.path.join(source, next_path_part)
new_target = format("{target}/{next_path_part}")
if sudo.path_isdir(new_source):
Logger.info(format("Creating DFS directory {new_target}"))
self._create_directory(new_target)
self._copy_from_local_directory(new_target, new_source)
else:
self._create_file(new_target, new_source)
def copy_toolkit_scripts(toolkit_files_dir, toolkit_tmp_dir, user, group,
upgrade_type, service):
import params
if service == NIFI:
run_ca_tmp_script = os.path.join(toolkit_tmp_dir, 'run_ca.sh')
new_run_ca_tmp_script = StaticFile("run_ca.sh")
if not sudo.path_isfile(run_ca_tmp_script) or sudo.read_file(
run_ca_tmp_script) != new_run_ca_tmp_script:
File(format(run_ca_tmp_script),
content=new_run_ca_tmp_script,
mode=0755,
owner=user,
group=group)
if not params.stack_support_nifi_toolkit_package:
nifiToolkitDirFilesPath = None
nifiToolkitDirTmpPath = None
Logger.info("Toolkit files dir is " + toolkit_files_dir)
Logger.info("Toolkit tmp dir is " + toolkit_tmp_dir)
for dir in os.listdir(toolkit_files_dir):
if dir.startswith('nifi-toolkit-'):
nifiToolkitDirFilesPath = os.path.join(toolkit_files_dir, dir)
nifiToolkitDirTmpPath = os.path.join(toolkit_tmp_dir, dir)
if not sudo.path_isdir(nifiToolkitDirTmpPath) or not (upgrade_type is
None):
os.system("\cp -r " + nifiToolkitDirFilesPath + " " +
toolkit_tmp_dir)
Directory(nifiToolkitDirTmpPath,
owner=user,
group=group,
create_parents=False,
recursive_ownership=True,
cd_access="a",
mode=0755)
os.system("\/var/lib/ambari-agent/ambari-sudo.sh chmod -R 755 " +
nifiToolkitDirTmpPath)
else:
Logger.info("Changing owner of package files")
package_toolkit_dir = os.path.join(params.stack_root, 'current',
'nifi-toolkit')
Directory(package_toolkit_dir,
owner=user,
group=group,
create_parents=False,
recursive_ownership=True,
cd_access="a",
mode=0755,
recursion_follow_links=True)
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s" % self.resource)
if self.resource.recursive:
if self.resource.recursive_permission:
DirectoryProvider.makedirs_and_set_permission_recursively(path, self.resource.owner,
self.resource.group, self.resource.mode)
else:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
_ensure_metadata(path, self.resource.owner, self.resource.group,
mode=self.resource.mode, cd_access=self.resource.cd_access)
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
user_entity = group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
_user_entity = pwd.getpwnam(user)
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info(
"Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
if group:
_group_entity = grp.getgrnam(group)
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info(
"Changing group for %s from %d to %s" % (path, stat.st_gid, group))
sudo.chown(path, user_entity, group_entity)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" % (
path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access+"+rx")
dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
user_entity = group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
_user_entity = pwd.getpwnam(user)
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info("Changing owner for %s from %d to %s" %
(path, stat.st_uid, user))
if group:
_group_entity = grp.getgrnam(group)
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info("Changing group for %s from %d to %s" %
(path, stat.st_gid, group))
sudo.chown(path, user_entity, group_entity)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" %
(path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access + "+x")
dir_path = os.path.split(dir_path)[0]
def _prepare_tez_tarball():
"""
Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
It's very important to use the version of mapreduce which matches tez here.
Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the
GPL license has been accepted.
:return: the full path of the newly created tez tarball to use
"""
import tempfile
Logger.info("Preparing the Tez tarball...")
# get the mapreduce tarball which matches the version of tez
# tez installs the mapreduce tar, so it should always be present
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
_, tez_source_file, _, _ = get_tarball_paths("tez")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
tez_temp_dir = tempfile.mkdtemp(prefix="tez-tarball-", dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
sudo.chmod(tez_temp_dir, 0777)
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir))
tar_archive.untar_archive(tez_source_file, tez_temp_dir)
hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib",
"native")
tez_lib_dir = os.path.join(tez_temp_dir, "lib")
if not os.path.exists(hadoop_lib_native_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_dir))
if not os.path.exists(tez_lib_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist"
.format(tez_lib_dir))
# copy native libraries from hadoop to tez
Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo=True)
# if enabled, LZO GPL libraries must be copied as well
if lzo_utils.should_install_lzo():
stack_root = Script.get_stack_root()
service_version = component_version.get_component_repository_version(
service_name="TEZ")
# some installations might not have Tez, but MapReduce2 should be a fallback to get the LZO libraries from
if service_version is None:
Logger.warning(
"Tez does not appear to be installed, using the MapReduce version to get the LZO libraries"
)
service_version = component_version.get_component_repository_version(
service_name="MAPREDUCE2")
hadoop_lib_native_lzo_dir = os.path.join(stack_root, service_version,
"hadoop", "lib", "native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
Logger.warning(
"Unable to located native LZO libraries at {0}, falling back to hadoop home"
.format(hadoop_lib_native_lzo_dir))
hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current",
"hadoop-client", "lib",
"native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}"
.format(hadoop_lib_native_lzo_dir))
Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir),
sudo=True)
# ensure that the tez/lib directory is readable by non-root (which it typically is not)
Directory(tez_lib_dir, mode=0755, cd_access='a', recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
tez_native_tarball_staging_dir = os.path.join(
temp_dir, "tez-native-tarball-staging")
if not os.path.exists(tez_native_tarball_staging_dir):
Directory(tez_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
tez_tarball_with_native_lib = os.path.join(tez_native_tarball_staging_dir,
"tez-native.tar.gz")
Logger.info("Creating a new Tez tarball at {0}".format(
tez_tarball_with_native_lib))
# tar up Tez, making sure to specify nothing for the arcname so that it does not include an absolute path
with closing(tarfile.open(tez_tarball_with_native_lib,
"w:gz")) as new_tez_tarball:
new_tez_tarball.add(tez_temp_dir, arcname=os.path.sep)
# ensure that the tarball can be read and uploaded
sudo.chmod(tez_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
sudo.rmtree(tez_temp_dir)
return tez_tarball_with_native_lib
# defaults set to current based on role
hadoop_mapr_home = format("{stack_root}/current/{mapred_role_root}")
hadoop_yarn_home = format("{stack_root}/current/{yarn_role_root}")
# try to render the specific version
version = component_version.get_component_repository_version()
if version is None:
version = default("/commandParams/version", None)
if version is not None:
hadoop_mapr_versioned_home = format(
"{stack_root}/{version}/hadoop-mapreduce")
hadoop_yarn_versioned_home = format(
"{stack_root}/{version}/hadoop-yarn")
if sudo.path_isdir(hadoop_mapr_versioned_home):
hadoop_mapr_home = hadoop_mapr_versioned_home
if sudo.path_isdir(hadoop_yarn_versioned_home):
hadoop_yarn_home = hadoop_yarn_versioned_home
hadoop_mapred2_jar_location = hadoop_mapr_home
mapred_bin = format("{hadoop_mapr_home}/sbin")
yarn_bin = format("{hadoop_yarn_home}/sbin")
yarn_container_bin = format("{hadoop_yarn_home}/bin")
if stack_supports_timeline_state_store:
# Timeline Service property that was added timeline_state_store stack feature
ats_leveldb_state_store_dir = default(
'/configurations/yarn-site/yarn.timeline-service.leveldb-state-store.path',
def _ensure_metadata(path,
user,
group,
mode=None,
cd_access=None,
recursive_ownership=False,
recursive_mode_flags=None,
recursion_follow_links=False,
safemode_folders=[]):
user_entity = group_entity = None
_user_entity = _group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
try:
_user_entity = pwd.getpwnam(user)
except KeyError:
raise Fail("User '{0}' doesn't exist".format(user))
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info("Changing owner for %s from %d to %s" %
(path, stat.st_uid, user))
if group:
try:
_group_entity = grp.getgrnam(group)
except KeyError:
raise Fail("Group '{0}' doesn't exist".format(group))
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info("Changing group for %s from %d to %s" %
(path, stat.st_gid, group))
if recursive_ownership:
assert_not_safemode_folder(path, safemode_folders)
sudo.chown_recursive(path, _user_entity, _group_entity,
recursion_follow_links)
sudo.chown(path, user_entity, group_entity)
if recursive_mode_flags:
if not isinstance(recursive_mode_flags, dict):
raise Fail(
"'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)"
)
regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+")
for key, flags in recursive_mode_flags.iteritems():
if key != 'd' and key != 'f':
raise Fail(
"'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid"
% (str(recursive_mode_flags), str(key)))
if not re.match(regexp_to_match, flags):
raise Fail(
"'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]."
% (str(flags)))
assert_not_safemode_folder(path, safemode_folders)
sudo.chmod_recursive(path, recursive_mode_flags,
recursion_follow_links)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" %
(path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = re.sub('/+', '/', path)
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access + "+rx")
dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_ownership=False, recursive_mode_flags=None, recursion_follow_links=False, safemode_folders=[]):
user_entity = group_entity = None
_user_entity = _group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
try:
_user_entity = pwd.getpwnam(user)
except KeyError:
raise Fail("User '{0}' doesn't exist".format(user))
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info(
"Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
if group:
try:
_group_entity = grp.getgrnam(group)
except KeyError:
raise Fail("Group '{0}' doesn't exist".format(group))
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info(
"Changing group for %s from %d to %s" % (path, stat.st_gid, group))
if recursive_ownership:
assert_not_safemode_folder(path, safemode_folders)
sudo.chown_recursive(path, _user_entity, _group_entity, recursion_follow_links)
sudo.chown(path, user_entity, group_entity)
if recursive_mode_flags:
if not isinstance(recursive_mode_flags, dict):
raise Fail("'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)")
regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+" )
for key, flags in recursive_mode_flags.iteritems():
if key != 'd' and key != 'f':
raise Fail("'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid" % (str(recursive_mode_flags), str(key)))
if not re.match(regexp_to_match, flags):
raise Fail("'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]." % (str(flags)))
assert_not_safemode_folder(path, safemode_folders)
sudo.chmod_recursive(path, recursive_mode_flags, recursion_follow_links)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" % (
path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = re.sub('/+', '/', path)
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access+"+rx")
dir_path = os.path.split(dir_path)[0]
