def select_all(version_to_select):
"""
Executes <stack-selector-tool> on every component for the specified version. If the value passed in is a
stack version such as "2.3", then this will find the latest installed version which
could be "2.3.0.0-9999". If a version is specified instead, such as 2.3.0.0-1234, it will use
that exact version.
:param version_to_select: the version to <stack-selector-tool> on, such as "2.3" or "2.3.0.0-1234"
"""
stack_root = Script.get_stack_root()
(stack_selector_name, stack_selector_path,
stack_selector_package) = stack_tools.get_stack_tool(
stack_tools.STACK_SELECTOR_NAME)
# it's an error, but it shouldn't really stop anything from working
if version_to_select is None:
Logger.error(
format(
"Unable to execute {stack_selector_name} after installing because there was no version specified"
))
return
Logger.info("Executing {0} set all on {1}".format(stack_selector_name,
version_to_select))
command = format(
'{sudo} {stack_selector_path} set all `ambari-python-wrap {stack_selector_path} versions | grep ^{version_to_select} | tail -1`'
)
only_if_command = format('ls -d {stack_root}/{version_to_select}*')
Execute(command, only_if=only_if_command)
def get_component_version(stack_name, component_name):
"""
For any stack name, returns the version currently installed for a given component.
Because each stack name may have different logic, the input is a generic dictionary.
:param stack_name: one of HDP, HDPWIN, BIGTOP, PHD, etc. usually retrieved from
the command-#.json file's ["hostLevelParams"]["stack_name"]
:param component_name: Component name as a string necessary to get the version
:return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
"""
version = None
if stack_name is None or component_name is None:
Logger.error("Could not determine component version because of the parameters is empty. " \
"stack_name: %s, component_name: %s" % (str(stack_name), str(component_name)))
return version
out = None
code = -1
if not stack_name:
Logger.error("Stack name not provided")
elif not component_name:
Logger.error("Component name not provided")
else:
(stack_selector_name, stack_selector_path,
stack_selector_package) = stack_tools.get_stack_tool(
stack_tools.STACK_SELECTOR_NAME)
if stack_selector_name and stack_selector_path and os.path.exists(
stack_selector_path):
tmpfile = tempfile.NamedTemporaryFile()
get_stack_comp_version_cmd = ""
try:
# This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
with open(tmpfile.name, 'r') as file:
get_stack_comp_version_cmd = '%s status %s > %s' % (
stack_selector_path, component_name, tmpfile.name)
code, stdoutdata = shell.call(get_stack_comp_version_cmd,
quiet=True)
out = file.read()
if code != 0 or out is None:
raise Exception("Code is nonzero or output is empty")
Logger.debug("Command: %s\nOutput: %s" %
(get_stack_comp_version_cmd, str(out)))
matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
version = matches[0][0].strip() if matches and len(
matches) > 0 and len(matches[0]) > 0 else None
Logger.debug("Version for component %s: %s" %
(component_name, str(version)))
except Exception, e:
Logger.error(
"Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s."
% (component_name, get_stack_comp_version_cmd, str(code),
str(out)))
else:
def get_component_version_from_symlink(stack_name, component_name):
"""
Gets the version of the specified component by invoking the stack-select tool to query for the
version which is referenced by the symlink.
:param stack_name: one of HDP, HDPWIN, BIGTOP, PHD, etc. usually retrieved from
the command-#.json file's ["clusterLevelParams"]["stack_name"]
:param component_name: Component name as a string necessary to get the version
:return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
"""
version = None
if not stack_name or not component_name:
Logger.error("Could not determine component version because of the parameters is empty. " \
"stack_name: %s, component_name: %s" % (str(stack_name), str(component_name)))
return version
stack_selector_name, stack_selector_path, stack_selector_package = stack_tools.get_stack_tool(
stack_tools.STACK_SELECTOR_NAME)
if stack_selector_name and stack_selector_path and os.path.exists(
stack_selector_path):
tmpfile = tempfile.NamedTemporaryFile(delete=True)
out, code = None, -1
get_stack_comp_version_cmd = '%s status %s > %s' % (
stack_selector_path, component_name, tmpfile.name)
try:
# This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
# ToDo: original problem looks strange
with open(tmpfile.name, 'r') as f:
code = shell.call(get_stack_comp_version_cmd, quiet=True)[0]
out = f.read()
if code != 0 or out is None:
raise ValueError("Code is nonzero or output is empty")
Logger.debug("Command: %s\nOutput: %s" %
(get_stack_comp_version_cmd, str(out)))
matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
version = matches[0][0].strip(
) if matches and len(matches) > 0 and len(matches[0]) > 0 else None
Logger.debug("Version for component %s: %s" %
(component_name, str(version)))
except Exception, e:
Logger.error(
"Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s."
% (component_name, get_stack_comp_version_cmd, str(code),
str(out)))
def install(self, env):
print "Install"
self.prepare()
component_name = self.get_component_name()
repo_info = str(default("/hostLevelParams/repo_info", "1.1.1.1-1"))
matches = re.findall(r"([\d\.]+\-\d+)", repo_info)
version = matches[0] if matches and len(matches) > 0 else "1.1.1.1-1"
from resource_management.libraries.functions import stack_tools
(stack_selector_name, stack_selector_path,
stack_selector_package) = stack_tools.get_stack_tool(
stack_tools.STACK_SELECTOR_NAME)
command = 'ambari-python-wrap {0} install {1}'.format(
stack_selector_path, version)
Execute(command)
if component_name:
conf_select.select("PERF", component_name, version)
stack_select.select(component_name, version)
def select_all(version_to_select):
"""
Executes <stack-selector-tool> on every component for the specified version. If the value passed in is a
stack version such as "2.3", then this will find the latest installed version which
could be "2.3.0.0-9999". If a version is specified instead, such as 2.3.0.0-1234, it will use
that exact version.
:param version_to_select: the version to <stack-selector-tool> on, such as "2.3" or "2.3.0.0-1234"
"""
stack_root = Script.get_stack_root()
(stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
# it's an error, but it shouldn't really stop anything from working
if version_to_select is None:
Logger.error(format("Unable to execute {stack_selector_name} after installing because there was no version specified"))
return
Logger.info("Executing {0} set all on {1}".format(stack_selector_name, version_to_select))
command = format('{sudo} {stack_selector_path} set all `ambari-python-wrap {stack_selector_path} versions | grep ^{version_to_select} | tail -1`')
only_if_command = format('ls -d {stack_root}/{version_to_select}*')
Execute(command, only_if = only_if_command)
def copy_tarballs_to_hdfs(tarball_prefix,
stack_select_component_name,
component_user,
file_owner,
group_owner,
ignore_sysprep=False):
"""
:param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
:param stack_select_component_name: Component name to get the status to determine the version
:param component_user: User that will execute the Hadoop commands, usually smokeuser
:param file_owner: Owner of the files copied to HDFS (typically hdfs user)
:param group_owner: Group owner of the files copied to HDFS (typically hadoop group)
:param ignore_sysprep: Ignore sysprep directives
:return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception.
In order to call this function, params.py must have all of the following,
stack_version_formatted, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab,
hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function.
"""
import params
if not ignore_sysprep and hasattr(
params, "host_sys_prepped") and params.host_sys_prepped:
Logger.info(
"Host is sys-prepped. Tarball %s will not be copied for %s." %
(tarball_prefix, stack_select_component_name))
return 0
if not hasattr(params, "stack_version_formatted"
) or params.stack_version_formatted is None:
Logger.warning("Could not find stack_version_formatted")
return 1
component_tar_source_file, component_tar_destination_folder = _get_tar_source_and_dest_folder(
tarball_prefix)
if not component_tar_source_file or not component_tar_destination_folder:
Logger.warning(
"Could not retrieve properties for tarball with prefix: %s" %
str(tarball_prefix))
return 1
if not os.path.exists(component_tar_source_file):
Logger.warning("Could not find file: %s" %
str(component_tar_source_file))
return 1
# Ubuntu returns: "stdin: is not a tty", as subprocess32 output.
tmpfile = tempfile.NamedTemporaryFile()
out = None
(stack_selector_name, stack_selector_path,
stack_selector_package) = stack_tools.get_stack_tool(
stack_tools.STACK_SELECTOR_NAME)
with open(tmpfile.name, 'r+') as file:
get_stack_version_cmd = '%s status %s > %s' % (
stack_selector_path, stack_select_component_name, tmpfile.name)
code, stdoutdata = shell.call(get_stack_version_cmd)
out = file.read()
pass
if code != 0 or out is None:
Logger.warning(
"Could not verify stack version by calling '%s'. Return Code: %s, Output: %s."
% (get_stack_version_cmd, str(code), str(out)))
return 1
matches = re.findall(r"([\d\.]+(?:-\d+)?)", out)
stack_version = matches[0] if matches and len(matches) > 0 else None
if not stack_version:
Logger.error("Could not parse stack version from output of %s: %s" %
(stack_selector_name, str(out)))
return 1
file_name = os.path.basename(component_tar_source_file)
destination_file = os.path.join(component_tar_destination_folder,
file_name)
destination_file = destination_file.replace(
"{{ stack_version_formatted }}", stack_version)
does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file
kinit_if_needed = ""
if params.security_enabled:
kinit_if_needed = format(
"{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
if kinit_if_needed:
Execute(kinit_if_needed, user=component_user, path='/bin')
does_hdfs_file_exist = False
try:
ExecuteHadoop(does_hdfs_file_exist_cmd,
user=component_user,
logoutput=True,
conf_dir=params.hadoop_conf_dir,
bin_dir=params.hadoop_bin_dir)
does_hdfs_file_exist = True
except Fail:
pass
if not does_hdfs_file_exist:
source_and_dest_pairs = [
(component_tar_source_file, destination_file),
]
return _copy_files(source_and_dest_pairs, component_user, file_owner,
group_owner, kinit_if_needed)
return 1
def copy_tarballs_to_hdfs(tarball_prefix, stack_select_component_name, component_user, file_owner, group_owner, ignore_sysprep=False):
"""
:param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig
:param stack_select_component_name: Component name to get the status to determine the version
:param component_user: User that will execute the Hadoop commands, usually smokeuser
:param file_owner: Owner of the files copied to HDFS (typically hdfs user)
:param group_owner: Group owner of the files copied to HDFS (typically hadoop group)
:param ignore_sysprep: Ignore sysprep directives
:return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception.
In order to call this function, params.py must have all of the following,
stack_version_formatted, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab,
hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function.
"""
import params
if not ignore_sysprep and hasattr(params, "host_sys_prepped") and params.host_sys_prepped:
Logger.info("Host is sys-prepped. Tarball %s will not be copied for %s." % (tarball_prefix, stack_select_component_name))
return 0
if not hasattr(params, "stack_version_formatted") or params.stack_version_formatted is None:
Logger.warning("Could not find stack_version_formatted")
return 1
component_tar_source_file, component_tar_destination_folder = _get_tar_source_and_dest_folder(tarball_prefix)
if not component_tar_source_file or not component_tar_destination_folder:
Logger.warning("Could not retrieve properties for tarball with prefix: %s" % str(tarball_prefix))
return 1
if not os.path.exists(component_tar_source_file):
Logger.warning("Could not find file: %s" % str(component_tar_source_file))
return 1
# Ubuntu returns: "stdin: is not a tty", as subprocess output.
tmpfile = tempfile.NamedTemporaryFile()
out = None
(stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
with open(tmpfile.name, 'r+') as file:
get_stack_version_cmd = '%s status %s > %s' % (stack_selector_path, stack_select_component_name, tmpfile.name)
code, stdoutdata = shell.call(get_stack_version_cmd)
out = file.read()
pass
if code != 0 or out is None:
Logger.warning("Could not verify stack version by calling '%s'. Return Code: %s, Output: %s." %
(get_stack_version_cmd, str(code), str(out)))
return 1
matches = re.findall(r"([\d\.]+\-\d+)", out)
stack_version = matches[0] if matches and len(matches) > 0 else None
if not stack_version:
Logger.error("Could not parse stack version from output of %s: %s" % (stack_selector_name, str(out)))
return 1
file_name = os.path.basename(component_tar_source_file)
destination_file = os.path.join(component_tar_destination_folder, file_name)
destination_file = destination_file.replace("{{ stack_version_formatted }}", stack_version)
does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file
kinit_if_needed = ""
if params.security_enabled:
kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};")
if kinit_if_needed:
Execute(kinit_if_needed,
user=component_user,
path='/bin'
)
does_hdfs_file_exist = False
try:
ExecuteHadoop(does_hdfs_file_exist_cmd,
user=component_user,
logoutput=True,
conf_dir=params.hadoop_conf_dir,
bin_dir=params.hadoop_bin_dir
)
does_hdfs_file_exist = True
except Fail:
pass
if not does_hdfs_file_exist:
source_and_dest_pairs = [(component_tar_source_file, destination_file), ]
return _copy_files(source_and_dest_pairs, component_user, file_owner, group_owner, kinit_if_needed)
return 1
