def _ensure_metadata(path, user, group, mode=None, cd_access=None):
stat = sudo.stat(path)
if user:
uid = _coerce_uid(user)
if stat.st_uid != uid:
Logger.info("Changing owner for %s from %d to %s" %
(path, stat.st_uid, user))
sudo.chown(path, user, None)
if group:
gid = _coerce_gid(group)
if stat.st_gid != gid:
Logger.info("Changing group for %s from %d to %s" %
(path, stat.st_gid, group))
sudo.chown(path, None, group)
if mode:
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" %
(path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access + "+x")
dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
stat = sudo.stat(path)
if user:
uid = _coerce_uid(user)
if stat.st_uid != uid:
Logger.info(
"Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
sudo.chown(path, user, None)
if group:
gid = _coerce_gid(group)
if stat.st_gid != gid:
Logger.info(
"Changing group for %s from %d to %s" % (path, stat.st_gid, group))
sudo.chown(path, None, group)
if mode:
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" % (
path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access+"+x")
dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
user_entity = group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
_user_entity = pwd.getpwnam(user)
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info("Changing owner for %s from %d to %s" %
(path, stat.st_uid, user))
if group:
_group_entity = grp.getgrnam(group)
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info("Changing group for %s from %d to %s" %
(path, stat.st_gid, group))
sudo.chown(path, user_entity, group_entity)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" %
(path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access + "+x")
dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None):
user_entity = group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
_user_entity = pwd.getpwnam(user)
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info(
"Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
if group:
_group_entity = grp.getgrnam(group)
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info(
"Changing group for %s from %d to %s" % (path, stat.st_gid, group))
sudo.chown(path, user_entity, group_entity)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" % (
path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = path
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access+"+rx")
dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path,
user,
group,
mode=None,
cd_access=None,
recursive_ownership=False,
recursive_mode_flags=None,
recursion_follow_links=False,
safemode_folders=[]):
user_entity = group_entity = None
_user_entity = _group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
try:
_user_entity = pwd.getpwnam(user)
except KeyError:
raise Fail("User '{0}' doesn't exist".format(user))
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info("Changing owner for %s from %d to %s" %
(path, stat.st_uid, user))
if group:
try:
_group_entity = grp.getgrnam(group)
except KeyError:
raise Fail("Group '{0}' doesn't exist".format(group))
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info("Changing group for %s from %d to %s" %
(path, stat.st_gid, group))
if recursive_ownership:
assert_not_safemode_folder(path, safemode_folders)
sudo.chown_recursive(path, _user_entity, _group_entity,
recursion_follow_links)
sudo.chown(path, user_entity, group_entity)
if recursive_mode_flags:
if not isinstance(recursive_mode_flags, dict):
raise Fail(
"'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)"
)
regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+")
for key, flags in recursive_mode_flags.iteritems():
if key != 'd' and key != 'f':
raise Fail(
"'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid"
% (str(recursive_mode_flags), str(key)))
if not re.match(regexp_to_match, flags):
raise Fail(
"'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]."
% (str(flags)))
assert_not_safemode_folder(path, safemode_folders)
sudo.chmod_recursive(path, recursive_mode_flags,
recursion_follow_links)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" %
(path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = re.sub('/+', '/', path)
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access + "+rx")
dir_path = os.path.split(dir_path)[0]
def prepare_libext_directory():
"""
Performs the following actions on libext:
- creates /usr/hdp/current/oozie/libext and recursively
- set 777 permissions on it and its parents.
- downloads JDBC driver JAR if needed
- copies Falcon JAR for the Oozie WAR if needed
"""
import params
# some versions of HDP don't need the lzo compression libraries
target_version_needs_compression_libraries = compare_versions(
format_hdp_stack_version(params.version), '2.2.1.0') >= 0
# ensure the directory exists
Directory(params.oozie_libext_dir, mode = 0777)
# get all hadooplzo* JAR files
# hdp-select set hadoop-client has not run yet, therefore we cannot use
# /usr/hdp/current/hadoop-client ; we must use params.version directly
# however, this only works when upgrading beyond 2.2.0.0; don't do this
# for downgrade to 2.2.0.0 since hadoop-lzo will not be present
# This can also be called during a Downgrade.
# When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
# if lzo is enabled.
if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
hadoop_lzo_pattern = 'hadoop-lzo*.jar'
hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
if not files:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy files into libext
files_copied = False
for file in files:
if os.path.isfile(file):
Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir))
shutil.copy2(file, params.oozie_libext_dir)
files_copied = True
if not files_copied:
raise Fail("There are no files at {0} matching {1}".format(
hadoop_client_new_lib_dir, hadoop_lzo_pattern))
# copy ext ZIP to libext dir
oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
# something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip
oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip")
if not os.path.isfile(oozie_ext_zip_file):
raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir))
Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True)
Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True)
sudo.chmod(oozie_ext_zip_target_path, 0644)
# Redownload jdbc driver to a new current location
oozie.download_database_library_if_needed()
# get the upgrade version in the event that it's needed
upgrade_stack = hdp_select._get_upgrade_stack()
if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
# something like 2.3.0.0-1234
stack_version = upgrade_stack[1]
# copy the Falcon JAR if needed; falcon has not upgraded yet, so we must
# use the versioned falcon directory
if params.has_falcon_host:
versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version)
Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir))
Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}'))
Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_ownership=False, recursive_mode_flags=None, recursion_follow_links=False, safemode_folders=[]):
user_entity = group_entity = None
_user_entity = _group_entity = None
if user or group:
stat = sudo.stat(path)
if user:
try:
_user_entity = pwd.getpwnam(user)
except KeyError:
raise Fail("User '{0}' doesn't exist".format(user))
if stat.st_uid != _user_entity.pw_uid:
user_entity = _user_entity
Logger.info(
"Changing owner for %s from %d to %s" % (path, stat.st_uid, user))
if group:
try:
_group_entity = grp.getgrnam(group)
except KeyError:
raise Fail("Group '{0}' doesn't exist".format(group))
if stat.st_gid != _group_entity.gr_gid:
group_entity = _group_entity
Logger.info(
"Changing group for %s from %d to %s" % (path, stat.st_gid, group))
if recursive_ownership:
assert_not_safemode_folder(path, safemode_folders)
sudo.chown_recursive(path, _user_entity, _group_entity, recursion_follow_links)
sudo.chown(path, user_entity, group_entity)
if recursive_mode_flags:
if not isinstance(recursive_mode_flags, dict):
raise Fail("'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)")
regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+" )
for key, flags in recursive_mode_flags.iteritems():
if key != 'd' and key != 'f':
raise Fail("'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid" % (str(recursive_mode_flags), str(key)))
if not re.match(regexp_to_match, flags):
raise Fail("'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]." % (str(flags)))
assert_not_safemode_folder(path, safemode_folders)
sudo.chmod_recursive(path, recursive_mode_flags, recursion_follow_links)
if mode:
stat = sudo.stat(path)
if stat.st_mode != mode:
Logger.info("Changing permission for %s from %o to %o" % (
path, stat.st_mode, mode))
sudo.chmod(path, mode)
if cd_access:
if not re.match("^[ugoa]+$", cd_access):
raise Fail("'cd_acess' value '%s' is not valid" % (cd_access))
dir_path = re.sub('/+', '/', path)
while dir_path != os.sep:
if sudo.path_isdir(dir_path):
sudo.chmod_extended(dir_path, cd_access+"+rx")
dir_path = os.path.split(dir_path)[0]
