def action_create(self):
path = self.resource.path
if sudo.path_lexists(path):
oldpath = os.path.realpath(path)
if oldpath == self.resource.to:
return
if not sudo.path_lexists(path):
raise Fail(
"%s trying to create a symlink with the same name as an existing file or directory"
% self.resource)
Logger.info("%s replacing old symlink to %s" %
(self.resource, oldpath))
sudo.unlink(path)
if self.resource.hard:
if not sudo.path_exists(self.resource.to):
raise Fail(
"Failed to apply %s, linking to nonexistent location %s" %
(self.resource, self.resource.to))
if sudo.path_isdir(self.resource.to):
raise Fail(
"Failed to apply %s, cannot create hard link to a directory (%s)"
% (self.resource, self.resource.to))
Logger.info("Creating hard %s" % self.resource)
sudo.link(self.resource.to, path)
else:
if not sudo.path_exists(self.resource.to):
Logger.info("Warning: linking to nonexistent location %s" %
self.resource.to)
Logger.info("Creating symbolic %s to %s" %
(self.resource, self.resource.to))
sudo.symlink(self.resource.to, path)
def action_create(self):
path = self.resource.path
if sudo.path_lexists(path):
oldpath = os.path.realpath(path)
if oldpath == self.resource.to:
return
if not sudo.path_lexists(path):
raise Fail(
"%s trying to create a symlink with the same name as an existing file or directory" % self.resource)
Logger.info("%s replacing old symlink to %s" % (self.resource, oldpath))
sudo.unlink(path)
if self.resource.hard:
if not sudo.path_exists(self.resource.to):
raise Fail("Failed to apply %s, linking to nonexistent location %s" % (self.resource, self.resource.to))
if sudo.path_isdir(self.resource.to):
raise Fail("Failed to apply %s, cannot create hard link to a directory (%s)" % (self.resource, self.resource.to))
Logger.info("Creating hard %s" % self.resource)
sudo.link(self.resource.to, path)
else:
if not sudo.path_exists(self.resource.to):
Logger.info("Warning: linking to nonexistent location %s" % self.resource.to)
Logger.info("Creating symbolic %s to %s" % (self.resource, self.resource.to))
sudo.symlink(self.resource.to, path)
def _relink_configurations_with_conf_select(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info(
"Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info(
"Unrecognized stack id {0}, cannot create config links".format(
stack_id))
return
target_stack_version = args[1]
if not (target_stack_version and check_stack_feature(
StackFeature.CONFIG_VERSIONING, target_stack_version)):
Logger.info("Configuration symlinks are not needed for {0}".format(
stack_version))
return
# After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink
if self.stack_name.upper() == "HDF" \
and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"):
Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select")
restricted_packages = conf_select.get_restricted_packages()
if 0 == len(restricted_packages):
Logger.info(
"There are no restricted conf-select packages for this installation"
)
else:
Logger.info("Restricting conf-select packages to {0}".format(
restricted_packages))
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
if 0 == len(restricted_packages
) or package_name in restricted_packages:
conf_select.convert_conf_directories_to_symlinks(
package_name, stack_version, directories)
def action_run(self):
if self.resource.creates:
if sudo.path_exists(self.resource.creates):
Logger.info("Skipping %s due to creates" % self.resource)
return
env = self.resource.environment
for i in range (0, self.resource.tries):
try:
shell.checked_call(self.resource.command, logoutput=self.resource.logoutput,
cwd=self.resource.cwd, env=env,
preexec_fn=_preexec_fn(self.resource), user=self.resource.user,
wait_for_finish=self.resource.wait_for_finish,
timeout=self.resource.timeout,
path=self.resource.path,
sudo=self.resource.sudo,
on_new_line=self.resource.on_new_line)
break
except Fail as ex:
if i == self.resource.tries-1: # last try
raise ex
else:
Logger.info("Retrying after %d seconds. Reason: %s" % (self.resource.try_sleep, str(ex)))
time.sleep(self.resource.try_sleep)
except ExecuteTimeoutException:
err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (self.resource.command, self.resource.timeout)
if self.resource.on_timeout:
Logger.info("Executing '%s'. Reason: %s" % (self.resource.on_timeout, err_msg))
shell.checked_call(self.resource.on_timeout)
else:
raise Fail(err_msg)
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s since it doesn't exist." % self.resource)
# dead links should be followed, else we gonna have failures on trying to create directories on top of them.
if self.resource.follow:
# Follow symlink until the tail.
followed_links = set()
while sudo.path_lexists(path):
if path in followed_links:
raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path))
followed_links.add(path)
path = sudo.readlink(path)
if path != self.resource.path:
Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path))
if self.resource.create_parents:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
_ensure_metadata(path, self.resource.owner, self.resource.group,
mode=self.resource.mode, cd_access=self.resource.cd_access,
recursive_ownership=self.resource.recursive_ownership, recursive_mode_flags=self.resource.recursive_mode_flags,
recursion_follow_links=self.resource.recursion_follow_links, safemode_folders=self.resource.safemode_folders)
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s since it doesn't exist." % self.resource)
# dead links should be followed, else we gonna have failures on trying to create directories on top of them.
if self.resource.follow:
# Follow symlink until the tail.
followed_links = set()
while sudo.path_lexists(path):
if path in followed_links:
raise Fail("Applying %s failed, looped symbolic links found while resolving %s" % (self.resource, path))
followed_links.add(path)
path = sudo.readlink(path)
if path != self.resource.path:
Logger.info("Following the link {0} to {1} to create the directory".format(self.resource.path, path))
if self.resource.recursive:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
_ensure_metadata(path, self.resource.owner, self.resource.group,
mode=self.resource.mode, cd_access=self.resource.cd_access)
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env, action='start')
start_cmd = format("{ams_grafana_script} start")
Execute(
start_cmd,
user=params.ams_user,
not_if=params.grafana_process_exists_cmd,
)
pidfile = format("{ams_grafana_pid_dir}/grafana-server.pid")
if not sudo.path_exists(pidfile):
Logger.warning(
"Pid file doesn't exist after starting of the component.")
else:
Logger.info("Grafana Server has started with pid: {0}".format(
sudo.read_file(pidfile).strip()))
#Set Grafana admin pwd
create_grafana_admin_pwd()
# Create datasource
create_ams_datasource()
# Create pre-built dashboards
create_ams_dashboards()
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s" % self.resource)
if self.resource.recursive:
if self.resource.recursive_permission:
DirectoryProvider.makedirs_and_set_permission_recursively(
path, self.resource.owner, self.resource.group,
self.resource.mode)
else:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail(
"Applying %s failed, parent directory %s doesn't exist"
% (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" %
(self.resource, path))
_ensure_metadata(path,
self.resource.owner,
self.resource.group,
mode=self.resource.mode,
cd_access=self.resource.cd_access)
def action_create(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
write = False
content = self._get_content()
if not sudo.path_exists(path):
write = True
reason = "it doesn't exist"
elif self.resource.replace:
if content is not None:
old_content = sudo.read_file(path, encoding=self.resource.encoding)
if content != old_content:
write = True
reason = "contents don't match"
if self.resource.backup:
self.resource.env.backup_file(path)
if write:
Logger.info("Writing %s because %s" % (self.resource, reason))
sudo.create_file(path, content, encoding=self.resource.encoding)
_ensure_metadata(self.resource.path, self.resource.owner,
self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s since it doesn't exist." %
self.resource)
# dead links should be followed, else we gonna have failures on trying to create directories on top of them.
if self.resource.follow:
# Follow symlink until the tail.
followed_links = set()
while sudo.path_islink(path):
if path in followed_links:
raise Fail(
"Applying %s failed, looped symbolic links found while resolving %s"
% (self.resource, path))
followed_links.add(path)
prev_path = path
path = sudo.readlink(path)
if not os.path.isabs(path):
path = os.path.join(os.path.dirname(prev_path), path)
if path != self.resource.path:
Logger.info(
"Following the link {0} to {1} to create the directory"
.format(self.resource.path, path))
if self.resource.create_parents:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail(
"Applying %s failed, parent directory %s doesn't exist"
% (self.resource, dirname))
try:
sudo.makedir(path, self.resource.mode or 0755)
except Exception as ex:
# race condition (somebody created the file before us)
if "File exists" in str(ex):
sudo.makedirs(path, self.resource.mode or 0755)
else:
raise
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" %
(self.resource, path))
_ensure_metadata(
path,
self.resource.owner,
self.resource.group,
mode=self.resource.mode,
cd_access=self.resource.cd_access,
recursive_ownership=self.resource.recursive_ownership,
recursive_mode_flags=self.resource.recursive_mode_flags,
recursion_follow_links=self.resource.recursion_follow_links,
safemode_folders=self.resource.safemode_folders)
def action_create(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, directory with name %s exists" % (self.resource, path))
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
write = False
content = self._get_content()
if not sudo.path_exists(path):
write = True
reason = "it doesn't exist"
elif self.resource.replace:
if content is not None:
old_content = sudo.read_file(path, encoding=self.resource.encoding)
if content != old_content:
write = True
reason = "contents don't match"
if self.resource.backup:
self.resource.env.backup_file(path)
if write:
Logger.info("Writing %s because %s" % (self.resource, reason))
sudo.create_file(path, content, encoding=self.resource.encoding)
_ensure_metadata(self.resource.path, self.resource.owner,
self.resource.group, mode=self.resource.mode, cd_access=self.resource.cd_access)
def action_delete(self):
path = self.resource.path
if sudo.path_exists(path):
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
Logger.info("Removing directory %s and all its content" % self.resource)
sudo.rmtree(path)
def action_delete(self):
path = self.resource.path
if sudo.path_exists(path):
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
Logger.info("Removing directory %s and all its content" % self.resource)
sudo.rmtree(path)
def action_delete(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def action_delete(self):
path = self.resource.path
if sudo.path_isdir(path):
raise Fail("Applying %s failed, %s is directory not file!" % (self.resource, path))
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def _create_config_links_if_necessary(self, stack_id, stack_version):
"""
Sets up the required structure for /etc/<component>/conf symlinks and <stack-root>/current
configuration symlinks IFF the current stack is < HDP 2.3+ and the new stack is >= HDP 2.3
stack_id: stack id, ie HDP-2.3
stack_version: version to set, ie 2.3.0.0-1234
"""
if stack_id is None:
Logger.info(
"Cannot create config links when stack_id is not defined")
return
args = stack_id.upper().split('-')
if len(args) != 2:
Logger.info(
"Unrecognized stack id {0}, cannot create config links".format(
stack_id))
return
target_stack_version = args[1]
if not (target_stack_version and check_stack_feature(
StackFeature.CONFIG_VERSIONING, target_stack_version)):
Logger.info("Configuration symlinks are not needed for {0}".format(
stack_version))
return
# After upgrading hdf-select package from HDF-2.X to HDF-3.Y, we need to create this symlink
if self.stack_name.upper() == "HDF" \
and not sudo.path_exists("/usr/bin/conf-select") and sudo.path_exists("/usr/bin/hdfconf-select"):
Link("/usr/bin/conf-select", to="/usr/bin/hdfconf-select")
for package_name, directories in conf_select.get_package_dirs(
).iteritems():
conf_select.select(self.stack_name,
package_name,
stack_version,
ignore_errors=True)
def post_start(self, env=None):
pid_files = self.get_pid_files()
if pid_files == []:
Logger.logger.warning("Pid files for current script are not defined")
return
pids = []
for pid_file in pid_files:
if not sudo.path_exists(pid_file):
raise Fail("Pid file {0} doesn't exist after starting of the component.".format(pid_file))
pids.append(sudo.read_file(pid_file).strip())
Logger.info("Component has started with pid(s): {0}".format(', '.join(pids)))
def makedirs_and_set_permission_recursively(path, owner, group, mode):
folders = []
path, folder = os.path.split(path)
while folder != "":
folders.append(folder)
path, folder = os.path.split(path)
if path != "":
folders.append(path)
folders.reverse()
dir_prefix = ""
for folder in folders:
dir_prefix = os.path.join(dir_prefix, folder)
if not sudo.path_exists(dir_prefix):
sudo.makedir(dir_prefix, mode or 0755)
_ensure_metadata(dir_prefix, None, None, mode)
def makedirs_and_set_permission_recursively(path, owner, group, mode):
folders=[]
path,folder=os.path.split(path)
while folder!="":
folders.append(folder)
path,folder=os.path.split(path)
if path!="":
folders.append(path)
folders.reverse()
dir_prefix=""
for folder in folders:
dir_prefix=os.path.join(dir_prefix, folder)
if not sudo.path_exists(dir_prefix):
sudo.makedir(dir_prefix, mode or 0755)
_ensure_metadata(dir_prefix, None, None, mode)
def action_run(self):
if self.resource.creates:
if sudo.path_exists(self.resource.creates):
Logger.info("Skipping %s due to creates" % self.resource)
return
shell.checked_call(self.resource.command, logoutput=self.resource.logoutput,
cwd=self.resource.cwd, env=self.resource.environment,
preexec_fn=_preexec_fn(self.resource), user=self.resource.user,
wait_for_finish=self.resource.wait_for_finish,
timeout=self.resource.timeout,on_timeout=self.resource.on_timeout,
path=self.resource.path,
sudo=self.resource.sudo,
on_new_line=self.resource.on_new_line,
stdout=self.resource.stdout,stderr=self.resource.stderr,
tries=self.resource.tries, try_sleep=self.resource.try_sleep)
def action_run(self):
if self.resource.creates:
if sudo.path_exists(self.resource.creates):
Logger.info("Skipping %s due to creates" % self.resource)
return
shell.checked_call(self.resource.command, logoutput=self.resource.logoutput,
cwd=self.resource.cwd, env=self.resource.environment,
preexec_fn=_preexec_fn(self.resource), user=self.resource.user,
wait_for_finish=self.resource.wait_for_finish,
timeout=self.resource.timeout,on_timeout=self.resource.on_timeout,
path=self.resource.path,
sudo=self.resource.sudo,
on_new_line=self.resource.on_new_line,
stdout=self.resource.stdout,stderr=self.resource.stderr,
tries=self.resource.tries, try_sleep=self.resource.try_sleep)
def action_run(self):
if self.resource.creates:
if sudo.path_exists(self.resource.creates):
Logger.info("Skipping %s due to creates" % self.resource)
return
env = self.resource.environment
for i in range(0, self.resource.tries):
try:
shell.checked_call(
self.resource.command,
logoutput=self.resource.logoutput,
cwd=self.resource.cwd,
env=env,
preexec_fn=_preexec_fn(self.resource),
user=self.resource.user,
wait_for_finish=self.resource.wait_for_finish,
timeout=self.resource.timeout,
path=self.resource.path,
sudo=self.resource.sudo,
on_new_line=self.resource.on_new_line)
break
except Fail as ex:
if i == self.resource.tries - 1: # last try
raise ex
else:
Logger.info("Retrying after %d seconds. Reason: %s" %
(self.resource.try_sleep, str(ex)))
time.sleep(self.resource.try_sleep)
except ExecuteTimeoutException:
err_msg = (
"Execution of '%s' was killed due timeout after %d seconds"
) % (self.resource.command, self.resource.timeout)
if self.resource.on_timeout:
Logger.info("Executing '%s'. Reason: %s" %
(self.resource.on_timeout, err_msg))
shell.checked_call(self.resource.on_timeout)
else:
raise Fail(err_msg)
def action_create(self):
path = self.resource.path
if not sudo.path_exists(path):
Logger.info("Creating directory %s" % self.resource)
if self.resource.recursive:
if self.resource.recursive_permission:
DirectoryProvider.makedirs_and_set_permission_recursively(path, self.resource.owner,
self.resource.group, self.resource.mode)
else:
sudo.makedirs(path, self.resource.mode or 0755)
else:
dirname = os.path.dirname(path)
if not sudo.path_isdir(dirname):
raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
sudo.makedir(path, self.resource.mode or 0755)
if not sudo.path_isdir(path):
raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
_ensure_metadata(path, self.resource.owner, self.resource.group,
mode=self.resource.mode, cd_access=self.resource.cd_access)
def _prepare_mapreduce_tarball():
"""
Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO is
not enabled or has not been opted-in, then this will do nothing and return the original
tarball to upload to HDFS.
:return: the full path of the newly created mapreduce tarball to use or the original path
if no changes were made
"""
# get the mapreduce tarball to crack open and add LZO libraries to
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
if not lzo_utils.should_install_lzo():
return mapreduce_source_file
Logger.info("Preparing the mapreduce tarball with native LZO libraries...")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
# calculate the source directory for LZO
hadoop_lib_native_source_dir = os.path.join(
os.path.dirname(mapreduce_source_file), "lib", "native")
if not sudo.path_exists(hadoop_lib_native_source_dir):
raise Fail(
"Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_source_dir))
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib")
# copy native libraries from source hadoop to target
Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir),
sudo=True)
# ensure that the hadoop/lib/native directory is readable by non-root (which it typically is not)
Directory(mapreduce_lib_dir,
mode=0755,
cd_access='a',
recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
mapreduce_native_tarball_staging_dir = os.path.join(
temp_dir, "mapreduce-native-tarball-staging")
if not os.path.exists(mapreduce_native_tarball_staging_dir):
Directory(mapreduce_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
mapreduce_tarball_with_native_lib = os.path.join(
mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz")
Logger.info("Creating a new mapreduce tarball at {0}".format(
mapreduce_tarball_with_native_lib))
# tar up mapreduce, making sure to specify nothing for the arcname so that it does not include an absolute path
with closing(tarfile.open(mapreduce_tarball_with_native_lib,
"w:gz")) as new_tarball:
new_tarball.add(mapreduce_temp_dir, arcname=os.path.sep)
# ensure that the tarball can be read and uploaded
sudo.chmod(mapreduce_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
return mapreduce_tarball_with_native_lib
def action_delete(self):
path = self.resource.path
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
def keytab_exists(kerberos_record):
return sudo.path_exists(
get_property_value(kerberos_record, 'keytab_file_path'))
def action_delete(self):
path = self.resource.path
if sudo.path_exists(path):
Logger.info("Deleting %s" % self.resource)
sudo.unlink(path)
