def action_create(self): path = self.resource.path if sudo.path_lexists(path): oldpath = os.path.realpath(path) if oldpath == self.resource.to: return if not sudo.path_lexists(path): raise Fail( "%s trying to create a symlink with the same name as an existing file or directory" % self.resource) Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath)) sudo.unlink(path) if self.resource.hard: if not sudo.path_exists(self.resource.to): raise Fail( "Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to)) if sudo.path_isdir(self.resource.to): raise Fail( "Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to)) Logger.info("Creating hard %s" % self.resource) sudo.link(self.resource.to, path) else: if not sudo.path_exists(self.resource.to): Logger.info("Warning: linking to nonexistent location %s" % self.resource.to) Logger.info("Creating symbolic %s to %s" % (self.resource, self.resource.to)) sudo.symlink(self.resource.to, path)
def action_create(self): path = self.resource.path if sudo.path_lexists(path): oldpath = os.path.realpath(path) if oldpath == self.resource.to: return if not sudo.path_lexists(path): raise Fail( "%s trying to create a symlink with the same name as an existing file or directory" % self.resource) Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath)) sudo.unlink(path) if self.resource.hard: if not sudo.path_exists(self.resource.to): raise Fail("Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to)) if sudo.path_isdir(self.resource.to): raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to)) Logger.info("Creating hard %s" % self.resource) sudo.link(self.resource.to, path) else: if not sudo.path_exists(self.resource.to): Logger.info("Warning: linking to nonexistent location %s" % self.resource.to) Logger.info("Creating symbolic %s to %s" % (self.resource, self.resource.to)) sudo.symlink(self.resource.to, path)
def _relink_configurations_with_conf_select(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info( "Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info( "Unrecognized stack id {0}, cannot create config links".format( stack_id)) return target_stack_version = args[1] if not (target_stack_version and check_stack_feature( StackFeature.CONFIG_VERSIONING, target_stack_version)): Logger.info("Configuration symlinks are not needed for {0}".format( stack_version)) return # After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink if self.stack_name.upper() == "HDF" \ and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"): Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select") restricted_packages = conf_select.get_restricted_packages() if 0 == len(restricted_packages): Logger.info( "There are no restricted conf-select packages for this installation" ) else: Logger.info("Restricting conf-select packages to {0}".format( restricted_packages)) for package_name, directories in conf_select.get_package_dirs( ).iteritems(): if 0 == len(restricted_packages ) or package_name in restricted_packages: conf_select.convert_conf_directories_to_symlinks( package_name, stack_version, directories)
def action_run(self): if self.resource.creates: if sudo.path_exists(self.resource.creates): Logger.info("Skipping %s due to creates" % self.resource) return env = self.resource.environment for i in range (0, self.resource.tries): try: shell.checked_call(self.resource.command, logoutput=self.resource.logoutput, cwd=self.resource.cwd, env=env, preexec_fn=_preexec_fn(self.resource), user=self.resource.user, wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout, path=self.resource.path, sudo=self.resource.sudo, on_new_line=self.resource.on_new_line) break except Fail as ex: if i == self.resource.tries-1: # last try raise ex else: Logger.info("Retrying after %d seconds. Reason: %s" % (self.resource.try_sleep, str(ex))) time.sleep(self.resource.try_sleep) except ExecuteTimeoutException: err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (self.resource.command, self.resource.timeout) if self.resource.on_timeout: Logger.info("Executing '%s'. Reason: %s" % (self.resource.on_timeout, err_msg)) shell.checked_call(self.resource.on_timeout) else: raise Fail(err_msg)
def action_create(self): path = self.resource.path if not sudo.path_exists(path): Logger.info("Creating directory %s since it doesn't exist." % self.resource) # dead links should be followed, else we gonna have failures on trying to create directories on top of them. if self.resource.follow: # Follow symlink until the tail. followed_links = set() while sudo.path_lexists(path): if path in followed_links: raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path)) followed_links.add(path) path = sudo.readlink(path) if path != self.resource.path: Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path)) if self.resource.create_parents: sudo.makedirs(path, self.resource.mode or 0755) else: dirname = os.path.dirname(path) if not sudo.path_isdir(dirname): raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) sudo.makedir(path, self.resource.mode or 0755) if not sudo.path_isdir(path): raise Fail("Applying %s failed, file %s already exists" % (self.resource, path)) _ensure_metadata(path, self.resource.owner, self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access, recursive_ownership=self.resource.recursive_ownership, recursive_mode_flags=self.resource.recursive_mode_flags, recursion_follow_links=self.resource.recursion_follow_links, safemode_folders=self.resource.safemode_folders)
def action_create(self): path = self.resource.path if not sudo.path_exists(path): Logger.info("Creating directory %s since it doesn't exist." % self.resource) # dead links should be followed, else we gonna have failures on trying to create directories on top of them. if self.resource.follow: # Follow symlink until the tail. followed_links = set() while sudo.path_lexists(path): if path in followed_links: raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path)) followed_links.add(path) path = sudo.readlink(path) if path != self.resource.path: Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path)) if self.resource.recursive: sudo.makedirs(path, self.resource.mode or 0755) else: dirname = os.path.dirname(path) if not sudo.path_isdir(dirname): raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) sudo.makedir(path, self.resource.mode or 0755) if not sudo.path_isdir(path): raise Fail("Applying %s failed, file %s already exists" % (self.resource, path)) _ensure_metadata(path, self.resource.owner, self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def start(self, env, upgrade_type=None): import params env.set_params(params) self.configure(env, action='start') start_cmd = format("{ams_grafana_script} start") Execute( start_cmd, user=params.ams_user, not_if=params.grafana_process_exists_cmd, ) pidfile = format("{ams_grafana_pid_dir}/grafana-server.pid") if not sudo.path_exists(pidfile): Logger.warning( "Pid file doesn't exist after starting of the component.") else: Logger.info("Grafana Server has started with pid: {0}".format( sudo.read_file(pidfile).strip())) #Set Grafana admin pwd create_grafana_admin_pwd() # Create datasource create_ams_datasource() # Create pre-built dashboards create_ams_dashboards()
def action_create(self): path = self.resource.path if not sudo.path_exists(path): Logger.info("Creating directory %s" % self.resource) if self.resource.recursive: if self.resource.recursive_permission: DirectoryProvider.makedirs_and_set_permission_recursively( path, self.resource.owner, self.resource.group, self.resource.mode) else: sudo.makedirs(path, self.resource.mode or 0755) else: dirname = os.path.dirname(path) if not sudo.path_isdir(dirname): raise Fail( "Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) sudo.makedir(path, self.resource.mode or 0755) if not sudo.path_isdir(path): raise Fail("Applying %s failed, file %s already exists" % (self.resource, path)) _ensure_metadata(path, self.resource.owner, self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def action_create(self): path = self.resource.path if sudo.path_isdir(path): raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path)) dirname = os.path.dirname(path) if not sudo.path_isdir(dirname): raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) write = False content = self._get_content() if not sudo.path_exists(path): write = True reason = "it doesn't exist" elif self.resource.replace: if content is not None: old_content = sudo.read_file(path, encoding=self.resource.encoding) if content != old_content: write = True reason = "contents don't match" if self.resource.backup: self.resource.env.backup_file(path) if write: Logger.info("Writing %s because %s" % (self.resource, reason)) sudo.create_file(path, content, encoding=self.resource.encoding) _ensure_metadata(self.resource.path, self.resource.owner, self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def action_create(self): path = self.resource.path if not sudo.path_exists(path): Logger.info("Creating directory %s since it doesn't exist." % self.resource) # dead links should be followed, else we gonna have failures on trying to create directories on top of them. if self.resource.follow: # Follow symlink until the tail. followed_links = set() while sudo.path_islink(path): if path in followed_links: raise Fail( "Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path)) followed_links.add(path) prev_path = path path = sudo.readlink(path) if not os.path.isabs(path): path = os.path.join(os.path.dirname(prev_path), path) if path != self.resource.path: Logger.info( "Following the link {0} to {1} to create the directory" .format(self.resource.path, path)) if self.resource.create_parents: sudo.makedirs(path, self.resource.mode or 0755) else: dirname = os.path.dirname(path) if not sudo.path_isdir(dirname): raise Fail( "Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) try: sudo.makedir(path, self.resource.mode or 0755) except Exception as ex: # race condition (somebody created the file before us) if "File exists" in str(ex): sudo.makedirs(path, self.resource.mode or 0755) else: raise if not sudo.path_isdir(path): raise Fail("Applying %s failed, file %s already exists" % (self.resource, path)) _ensure_metadata( path, self.resource.owner, self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access, recursive_ownership=self.resource.recursive_ownership, recursive_mode_flags=self.resource.recursive_mode_flags, recursion_follow_links=self.resource.recursion_follow_links, safemode_folders=self.resource.safemode_folders)
def action_delete(self): path = self.resource.path if sudo.path_exists(path): if not sudo.path_isdir(path): raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path)) Logger.info("Removing directory %s and all its content" % self.resource) sudo.rmtree(path)
def action_delete(self): path = self.resource.path if sudo.path_isdir(path): raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path)) if sudo.path_exists(path): Logger.info("Deleting %s" % self.resource) sudo.unlink(path)
def _create_config_links_if_necessary(self, stack_id, stack_version): """ Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3 stack_id: stack id, ie HDP-2.3 stack_version: version to set, ie 2.3.0.0-1234 """ if stack_id is None: Logger.info( "Cannot create config links when stack_id is not defined") return args = stack_id.upper().split('-') if len(args) != 2: Logger.info( "Unrecognized stack id {0}, cannot create config links".format( stack_id)) return target_stack_version = args[1] if not (target_stack_version and check_stack_feature( StackFeature.CONFIG_VERSIONING, target_stack_version)): Logger.info("Configuration symlinks are not needed for {0}".format( stack_version)) return # After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink if self.stack_name.upper() == "HDF" \ and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"): Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select") for package_name, directories in conf_select.get_package_dirs( ).iteritems(): conf_select.select(self.stack_name, package_name, stack_version, ignore_errors=True)
def post_start(self, env=None): pid_files = self.get_pid_files() if pid_files == []: Logger.logger.warning("Pid files for current script are not defined") return pids = [] for pid_file in pid_files: if not sudo.path_exists(pid_file): raise Fail("Pid file {0} doesn't exist after starting of the component.".format(pid_file)) pids.append(sudo.read_file(pid_file).strip()) Logger.info("Component has started with pid(s): {0}".format(', '.join(pids)))
def makedirs_and_set_permission_recursively(path, owner, group, mode): folders = [] path, folder = os.path.split(path) while folder != "": folders.append(folder) path, folder = os.path.split(path) if path != "": folders.append(path) folders.reverse() dir_prefix = "" for folder in folders: dir_prefix = os.path.join(dir_prefix, folder) if not sudo.path_exists(dir_prefix): sudo.makedir(dir_prefix, mode or 0755) _ensure_metadata(dir_prefix, None, None, mode)
def makedirs_and_set_permission_recursively(path, owner, group, mode): folders=[] path,folder=os.path.split(path) while folder!="": folders.append(folder) path,folder=os.path.split(path) if path!="": folders.append(path) folders.reverse() dir_prefix="" for folder in folders: dir_prefix=os.path.join(dir_prefix, folder) if not sudo.path_exists(dir_prefix): sudo.makedir(dir_prefix, mode or 0755) _ensure_metadata(dir_prefix, None, None, mode)
def action_run(self): if self.resource.creates: if sudo.path_exists(self.resource.creates): Logger.info("Skipping %s due to creates" % self.resource) return shell.checked_call(self.resource.command, logoutput=self.resource.logoutput, cwd=self.resource.cwd, env=self.resource.environment, preexec_fn=_preexec_fn(self.resource), user=self.resource.user, wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout,on_timeout=self.resource.on_timeout, path=self.resource.path, sudo=self.resource.sudo, on_new_line=self.resource.on_new_line, stdout=self.resource.stdout,stderr=self.resource.stderr, tries=self.resource.tries, try_sleep=self.resource.try_sleep)
def action_run(self): if self.resource.creates: if sudo.path_exists(self.resource.creates): Logger.info("Skipping %s due to creates" % self.resource) return env = self.resource.environment for i in range(0, self.resource.tries): try: shell.checked_call( self.resource.command, logoutput=self.resource.logoutput, cwd=self.resource.cwd, env=env, preexec_fn=_preexec_fn(self.resource), user=self.resource.user, wait_for_finish=self.resource.wait_for_finish, timeout=self.resource.timeout, path=self.resource.path, sudo=self.resource.sudo, on_new_line=self.resource.on_new_line) break except Fail as ex: if i == self.resource.tries - 1: # last try raise ex else: Logger.info("Retrying after %d seconds. Reason: %s" % (self.resource.try_sleep, str(ex))) time.sleep(self.resource.try_sleep) except ExecuteTimeoutException: err_msg = ( "Execution of '%s' was killed due timeout after %d seconds" ) % (self.resource.command, self.resource.timeout) if self.resource.on_timeout: Logger.info("Executing '%s'. Reason: %s" % (self.resource.on_timeout, err_msg)) shell.checked_call(self.resource.on_timeout) else: raise Fail(err_msg)
def action_create(self): path = self.resource.path if not sudo.path_exists(path): Logger.info("Creating directory %s" % self.resource) if self.resource.recursive: if self.resource.recursive_permission: DirectoryProvider.makedirs_and_set_permission_recursively(path, self.resource.owner, self.resource.group, self.resource.mode) else: sudo.makedirs(path, self.resource.mode or 0755) else: dirname = os.path.dirname(path) if not sudo.path_isdir(dirname): raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname)) sudo.makedir(path, self.resource.mode or 0755) if not sudo.path_isdir(path): raise Fail("Applying %s failed, file %s already exists" % (self.resource, path)) _ensure_metadata(path, self.resource.owner, self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def _prepare_mapreduce_tarball(): """ Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO is not enabled or has not been opted-in, then this will do nothing and return the original tarball to upload to HDFS. :return: the full path of the newly created mapreduce tarball to use or the original path if no changes were made """ # get the mapreduce tarball to crack open and add LZO libraries to _, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce") if not lzo_utils.should_install_lzo(): return mapreduce_source_file Logger.info("Preparing the mapreduce tarball with native LZO libraries...") temp_dir = Script.get_tmp_dir() # create the temp staging directories ensuring that non-root agents using tarfile can work with them mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-", dir=temp_dir) sudo.chmod(mapreduce_temp_dir, 0777) # calculate the source directory for LZO hadoop_lib_native_source_dir = os.path.join( os.path.dirname(mapreduce_source_file), "lib", "native") if not sudo.path_exists(hadoop_lib_native_source_dir): raise Fail( "Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist" .format(hadoop_lib_native_source_dir)) Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir)) tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir) mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib") # copy native libraries from source hadoop to target Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir), sudo=True) # ensure that the hadoop/lib/native directory is readable by non-root (which it typically is not) Directory(mapreduce_lib_dir, mode=0755, cd_access='a', recursive_ownership=True) # create the staging directory so that non-root agents can write to it mapreduce_native_tarball_staging_dir = os.path.join( temp_dir, "mapreduce-native-tarball-staging") if not os.path.exists(mapreduce_native_tarball_staging_dir): Directory(mapreduce_native_tarball_staging_dir, mode=0777, cd_access='a', create_parents=True, recursive_ownership=True) mapreduce_tarball_with_native_lib = os.path.join( mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz") Logger.info("Creating a new mapreduce tarball at {0}".format( mapreduce_tarball_with_native_lib)) # tar up mapreduce, making sure to specify nothing for the arcname so that it does not include an absolute path with closing(tarfile.open(mapreduce_tarball_with_native_lib, "w:gz")) as new_tarball: new_tarball.add(mapreduce_temp_dir, arcname=os.path.sep) # ensure that the tarball can be read and uploaded sudo.chmod(mapreduce_tarball_with_native_lib, 0744) # cleanup sudo.rmtree(mapreduce_temp_dir) return mapreduce_tarball_with_native_lib
def action_delete(self): path = self.resource.path if sudo.path_exists(path): Logger.info("Deleting %s" % self.resource) sudo.unlink(path)
def keytab_exists(kerberos_record): return sudo.path_exists( get_property_value(kerberos_record, 'keytab_file_path'))