def _ensure_metadata(path, user, group, mode=None, cd_access=None): stat = sudo.stat(path) if user: uid = _coerce_uid(user) if stat.st_uid != uid: Logger.info("Changing owner for %s from %d to %s" % (path, stat.st_uid, user)) sudo.chown(path, user, None) if group: gid = _coerce_gid(group) if stat.st_gid != gid: Logger.info("Changing group for %s from %d to %s" % (path, stat.st_gid, group)) sudo.chown(path, None, group) if mode: if stat.st_mode != mode: Logger.info("Changing permission for %s from %o to %o" % (path, stat.st_mode, mode)) sudo.chmod(path, mode) if cd_access: if not re.match("^[ugoa]+$", cd_access): raise Fail("'cd_acess' value '%s' is not valid" % (cd_access)) dir_path = path while dir_path != os.sep: if sudo.path_isdir(dir_path): sudo.chmod_extended(dir_path, cd_access + "+x") dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None): stat = sudo.stat(path) if user: uid = _coerce_uid(user) if stat.st_uid != uid: Logger.info( "Changing owner for %s from %d to %s" % (path, stat.st_uid, user)) sudo.chown(path, user, None) if group: gid = _coerce_gid(group) if stat.st_gid != gid: Logger.info( "Changing group for %s from %d to %s" % (path, stat.st_gid, group)) sudo.chown(path, None, group) if mode: if stat.st_mode != mode: Logger.info("Changing permission for %s from %o to %o" % ( path, stat.st_mode, mode)) sudo.chmod(path, mode) if cd_access: if not re.match("^[ugoa]+$", cd_access): raise Fail("'cd_acess' value '%s' is not valid" % (cd_access)) dir_path = path while dir_path != os.sep: if sudo.path_isdir(dir_path): sudo.chmod_extended(dir_path, cd_access+"+x") dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None): user_entity = group_entity = None if user or group: stat = sudo.stat(path) if user: _user_entity = pwd.getpwnam(user) if stat.st_uid != _user_entity.pw_uid: user_entity = _user_entity Logger.info("Changing owner for %s from %d to %s" % (path, stat.st_uid, user)) if group: _group_entity = grp.getgrnam(group) if stat.st_gid != _group_entity.gr_gid: group_entity = _group_entity Logger.info("Changing group for %s from %d to %s" % (path, stat.st_gid, group)) sudo.chown(path, user_entity, group_entity) if mode: stat = sudo.stat(path) if stat.st_mode != mode: Logger.info("Changing permission for %s from %o to %o" % (path, stat.st_mode, mode)) sudo.chmod(path, mode) if cd_access: if not re.match("^[ugoa]+$", cd_access): raise Fail("'cd_acess' value '%s' is not valid" % (cd_access)) dir_path = path while dir_path != os.sep: if sudo.path_isdir(dir_path): sudo.chmod_extended(dir_path, cd_access + "+x") dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None): user_entity = group_entity = None if user or group: stat = sudo.stat(path) if user: _user_entity = pwd.getpwnam(user) if stat.st_uid != _user_entity.pw_uid: user_entity = _user_entity Logger.info( "Changing owner for %s from %d to %s" % (path, stat.st_uid, user)) if group: _group_entity = grp.getgrnam(group) if stat.st_gid != _group_entity.gr_gid: group_entity = _group_entity Logger.info( "Changing group for %s from %d to %s" % (path, stat.st_gid, group)) sudo.chown(path, user_entity, group_entity) if mode: stat = sudo.stat(path) if stat.st_mode != mode: Logger.info("Changing permission for %s from %o to %o" % ( path, stat.st_mode, mode)) sudo.chmod(path, mode) if cd_access: if not re.match("^[ugoa]+$", cd_access): raise Fail("'cd_acess' value '%s' is not valid" % (cd_access)) dir_path = path while dir_path != os.sep: if sudo.path_isdir(dir_path): sudo.chmod_extended(dir_path, cd_access+"+rx") dir_path = os.path.split(dir_path)[0]
def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_ownership=False, recursive_mode_flags=None, recursion_follow_links=False, safemode_folders=[]): user_entity = group_entity = None _user_entity = _group_entity = None if user or group: stat = sudo.stat(path) if user: try: _user_entity = pwd.getpwnam(user) except KeyError: raise Fail("User '{0}' doesn't exist".format(user)) if stat.st_uid != _user_entity.pw_uid: user_entity = _user_entity Logger.info("Changing owner for %s from %d to %s" % (path, stat.st_uid, user)) if group: try: _group_entity = grp.getgrnam(group) except KeyError: raise Fail("Group '{0}' doesn't exist".format(group)) if stat.st_gid != _group_entity.gr_gid: group_entity = _group_entity Logger.info("Changing group for %s from %d to %s" % (path, stat.st_gid, group)) if recursive_ownership: assert_not_safemode_folder(path, safemode_folders) sudo.chown_recursive(path, _user_entity, _group_entity, recursion_follow_links) sudo.chown(path, user_entity, group_entity) if recursive_mode_flags: if not isinstance(recursive_mode_flags, dict): raise Fail( "'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)" ) regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+") for key, flags in recursive_mode_flags.iteritems(): if key != 'd' and key != 'f': raise Fail( "'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid" % (str(recursive_mode_flags), str(key))) if not re.match(regexp_to_match, flags): raise Fail( "'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]." % (str(flags))) assert_not_safemode_folder(path, safemode_folders) sudo.chmod_recursive(path, recursive_mode_flags, recursion_follow_links) if mode: stat = sudo.stat(path) if stat.st_mode != mode: Logger.info("Changing permission for %s from %o to %o" % (path, stat.st_mode, mode)) sudo.chmod(path, mode) if cd_access: if not re.match("^[ugoa]+$", cd_access): raise Fail("'cd_acess' value '%s' is not valid" % (cd_access)) dir_path = re.sub('/+', '/', path) while dir_path != os.sep: if sudo.path_isdir(dir_path): sudo.chmod_extended(dir_path, cd_access + "+rx") dir_path = os.path.split(dir_path)[0]
def prepare_libext_directory(): """ Performs the following actions on libext: - creates /usr/hdp/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 # ensure the directory exists Directory(params.oozie_libext_dir, mode = 0777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to libext dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip") if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir)) Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) sudo.chmod(oozie_ext_zip_target_path, 0644) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = hdp_select._get_upgrade_stack() if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") # something like 2.3.0.0-1234 stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version) Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}')) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def _ensure_metadata(path, user, group, mode=None, cd_access=None, recursive_ownership=False, recursive_mode_flags=None, recursion_follow_links=False, safemode_folders=[]): user_entity = group_entity = None _user_entity = _group_entity = None if user or group: stat = sudo.stat(path) if user: try: _user_entity = pwd.getpwnam(user) except KeyError: raise Fail("User '{0}' doesn't exist".format(user)) if stat.st_uid != _user_entity.pw_uid: user_entity = _user_entity Logger.info( "Changing owner for %s from %d to %s" % (path, stat.st_uid, user)) if group: try: _group_entity = grp.getgrnam(group) except KeyError: raise Fail("Group '{0}' doesn't exist".format(group)) if stat.st_gid != _group_entity.gr_gid: group_entity = _group_entity Logger.info( "Changing group for %s from %d to %s" % (path, stat.st_gid, group)) if recursive_ownership: assert_not_safemode_folder(path, safemode_folders) sudo.chown_recursive(path, _user_entity, _group_entity, recursion_follow_links) sudo.chown(path, user_entity, group_entity) if recursive_mode_flags: if not isinstance(recursive_mode_flags, dict): raise Fail("'recursion_follow_links' value should be a dictionary with 'f' and(or) 'd' key (for file and directory permission flags)") regexp_to_match = "^({0},)*({0})$".format("[ugoa]+[+=-][rwx]+" ) for key, flags in recursive_mode_flags.iteritems(): if key != 'd' and key != 'f': raise Fail("'recursive_mode_flags' with value '%s' has unknown key '%s', only keys 'f' and 'd' are valid" % (str(recursive_mode_flags), str(key))) if not re.match(regexp_to_match, flags): raise Fail("'recursive_mode_flags' found '%s', but should value format have the following format: [ugoa...][[+-=][perms...]...]." % (str(flags))) assert_not_safemode_folder(path, safemode_folders) sudo.chmod_recursive(path, recursive_mode_flags, recursion_follow_links) if mode: stat = sudo.stat(path) if stat.st_mode != mode: Logger.info("Changing permission for %s from %o to %o" % ( path, stat.st_mode, mode)) sudo.chmod(path, mode) if cd_access: if not re.match("^[ugoa]+$", cd_access): raise Fail("'cd_acess' value '%s' is not valid" % (cd_access)) dir_path = re.sub('/+', '/', path) while dir_path != os.sep: if sudo.path_isdir(dir_path): sudo.chmod_extended(dir_path, cd_access+"+rx") dir_path = os.path.split(dir_path)[0]