def _get_single_version_from_hdp_select():
"""
Call "hdp-select versions" and return the version string if only one version is available.
:return: Returns a version string if successful, and None otherwise.
"""
# Ubuntu returns: "stdin: is not a tty", as subprocess output, so must use a temporary file to store the output.
tmpfile = tempfile.NamedTemporaryFile()
tmp_dir = Script.get_tmp_dir()
tmp_file = os.path.join(tmp_dir, "copy_tarball_out.txt")
hdp_version = None
out = None
get_hdp_versions_cmd = "/usr/bin/hdp-select versions > {0}".format(tmp_file)
try:
code, stdoutdata = shell.call(get_hdp_versions_cmd, logoutput=True)
with open(tmp_file, 'r+') as file:
out = file.read()
except Exception, e:
Logger.logger.exception("Could not parse output of {0}. Error: {1}".format(str(tmp_file), str(e)))
limitations under the License.
Ambari Agent
"""
import status_params
from resource_management.libraries.functions import format
from resource_management.libraries.functions.version import format_hdp_stack_version
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
stack_name = default("/hostLevelParams/stack_name", None)
current_version = default("/hostLevelParams/current_version", None)
component_directory = status_params.component_directory
# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
version = default("/commandParams/version", None)
# default parameters
zk_home = "/usr/hdp/2.3.2.0-2950/zookeeper"
zk_bin = "/usr/hdp/2.3.2.0-2950/zookeeper/bin"
zk_cli_shell = "/usr/hdp/2.3.2.0-2950/zookeeper/bin/zkCli.sh"
from resource_management import *
from resource_management.libraries.script.script import Script
import sys, os, glob, socket
# server configurations
config = Script.get_config()
service_packagedir = os.path.realpath(__file__).split('/scripts')[0]
tmp_dir = Script.get_tmp_dir()
kylin_properties = config['configurations']['kylin']['kylin_properties']
hdp_version = default("/commandParams/version", None)
downloadlocation = "http://mirror.bit.edu.cn/apache/kylin/apache-kylin-2.6.3/apache-kylin-2.6.3-bin-hadoop3.tar.gz"
#install_dir = config['configurations']['kylin']['install.dir']
install_dir = format('/opt/kylin')
current_host_name = socket.gethostname()
server_mode = "query"
server_masters = config['clusterHostInfo']['kylin_all_hosts'][0]
server_clusters_arr = config['clusterHostInfo']['kylin_all_hosts'] + (
config['clusterHostInfo'].has_key('kylin_query_hosts')
and config['clusterHostInfo']['kylin_query_hosts'] or [])
server_port = "7070"
server_clusters = ','.join(i + ":" + server_port for i in server_clusters_arr)
kylin_servers = ';'.join("server " + i + ":" + server_port
for i in server_clusters_arr) + ";"
hadoop_conf_dir = install_dir + "/latest/conf/hadoop_conf"
nginx_conf = config['configurations']['nginx']['nginx_conf']
nginx_port = config['configurations']['nginx']['nginx_port']
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
if params.hive_server_interactive_ha:
"""
Check llap app state
"""
Logger.info("HSI HA is enabled. Checking if LLAP is already running ...")
if params.stack_supports_hive_interactive_ga:
status = self.check_llap_app_status_in_llap_ga(params.llap_app_name, 2, params.hive_server_interactive_ha)
else:
status = self.check_llap_app_status_in_llap_tp(params.llap_app_name, 2, params.hive_server_interactive_ha)
if status:
Logger.info("LLAP app '{0}' is already running.".format(params.llap_app_name))
return True
else:
Logger.info("LLAP app '{0}' is not running. llap will be started.".format(params.llap_app_name))
pass
# Call for cleaning up the earlier run(s) LLAP package folders.
self._cleanup_past_llap_package_dirs()
Logger.info("Starting LLAP")
LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --slider-am-container-mb {params.slider_am_container_mb} "
"--size {params.llap_daemon_container_size}m --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m "
"--loglevel {params.llap_log_level} {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
# Append params that are supported from Hive llap GA version.
if params.stack_supports_hive_interactive_ga:
# Figure out the Slider Anti-affinity to be used.
# YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it
# starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP
# container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided.
slider_placement = 4
if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)):
slider_placement = 0
Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * "
"YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
else:
Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * "
"YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem))
cmd += format(" --slider-placement {slider_placement} --skiphadoopversion --skiphbasecp --instances {params.num_llap_daemon_running_nodes}")
# Setup the logger for the ga version only
cmd += format(" --logger {params.llap_logger}")
else:
cmd += format(" --instances {params.num_llap_nodes}")
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
"{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}")
# Add the aux jars if they are specified. If empty, dont need to add this param.
if params.hive_aux_jars:
cmd+= format(" --auxjars {params.hive_aux_jars}")
# Append args.
llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content()
cmd += format(" --args \" {llap_java_args}\"")
# Append metaspace size to args.
if params.java_version > 7 and params.llap_daemon_container_size > 4096:
if params.llap_daemon_container_size <= 32768:
metaspaceSize = "256m"
else:
metaspaceSize = "1024m"
cmd = cmd[:-1] + " -XX:MetaspaceSize="+metaspaceSize+ "\""
run_file_path = None
try:
Logger.info(format("LLAP start command: {cmd}"))
code, output, error = shell.checked_call(cmd, user=params.hive_user, quiet = True, stderr=subprocess.PIPE, logoutput=True)
if code != 0 or output is None:
raise Fail("Command failed with either non-zero return code or no output.")
# E.g., output:
# Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
exp = r"Prepared (.*?run.sh) for running LLAP"
run_file_path = None
out_splits = output.split("\n")
for line in out_splits:
line = line.strip()
m = re.match(exp, line, re.I)
if m and len(m.groups()) == 1:
run_file_name = m.group(1)
run_file_path = os.path.join(params.hive_user_home_dir, run_file_name)
break
if not run_file_path:
raise Fail("Did not find run.sh file in output: " + str(output))
Logger.info(format("Run file path: {run_file_path}"))
Execute(run_file_path, user=params.hive_user, logoutput=True)
Logger.info("Submitted LLAP app name : {0}".format(params.llap_app_name))
# We need to check the status of LLAP app to figure out it got
# launched properly and is in running state. Then go ahead with Hive Interactive Server start.
if params.stack_supports_hive_interactive_ga:
status = self.check_llap_app_status_in_llap_ga(params.llap_app_name, params.num_retries_for_checking_llap_status)
else:
status = self.check_llap_app_status_in_llap_tp(params.llap_app_name, params.num_retries_for_checking_llap_status)
if status:
Logger.info("LLAP app '{0}' deployed successfully.".format(params.llap_app_name))
return True
else:
Logger.error("LLAP app '{0}' deployment unsuccessful.".format(params.llap_app_name))
return False
except:
# Attempt to clean up the packaged application, or potentially rename it with a .bak
if run_file_path is not None and cleanup:
parent_dir = os.path.dirname(run_file_path)
Directory(parent_dir,
action = "delete",
ignore_failures = True,
)
# throw the original exception
raise
def _llap_start(self, env, cleanup=False):
import params
env.set_params(params)
Logger.info("Starting LLAP")
LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
LLAP_APP_NAME = 'llap0'
unique_name = "llap-slider%s" % datetime.utcnow().strftime(
'%Y-%m-%d_%H-%M-%S')
cmd = format(
"{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}"
" --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m "
" --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}"
" --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}")
if params.security_enabled:
llap_keytab_splits = params.hive_llap_keytab_file.split("/")
Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits))
cmd += format(
" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab "
"{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}"
)
# Add the aux jars if they are specified. If empty, dont need to add this param.
if params.hive_aux_jars:
cmd += format(" --auxjars {params.hive_aux_jars}")
# Append args.
llap_java_args = InlineTemplate(
params.llap_app_java_opts).get_content()
cmd += format(" --args \" {llap_java_args}\"")
run_file_path = None
try:
Logger.info(format("Command: {cmd}"))
code, output, error = shell.checked_call(cmd,
user=params.hive_user,
stderr=subprocess.PIPE,
logoutput=True)
if code != 0 or output is None:
raise Fail(
"Command failed with either non-zero return code or no output."
)
# E.g., output:
# Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider
exp = r"Prepared (.*?run.sh) for running LLAP"
run_file_path = None
out_splits = output.split("\n")
for line in out_splits:
line = line.strip()
m = re.match(exp, line, re.I)
if m and len(m.groups()) == 1:
run_file_name = m.group(1)
run_file_path = os.path.join(params.hive_user_home_dir,
run_file_name)
break
if not run_file_path:
raise Fail("Did not find run.sh file in output: " +
str(output))
Logger.info(format("Run file path: {run_file_path}"))
Execute(run_file_path, user=params.hive_user, logoutput=True)
Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME))
# We need to check the status of LLAP app to figure out it got
# launched properly and is in running state. Then go ahead with Hive Interactive Server start.
status = self.check_llap_app_status(
LLAP_APP_NAME, params.num_retries_for_checking_llap_status)
if status:
Logger.info("LLAP app '{0}' deployed successfully.".format(
LLAP_APP_NAME))
return True
else:
Logger.error("LLAP app '{0}' deployment unsuccessful.".format(
LLAP_APP_NAME))
return False
except:
# Attempt to clean up the packaged application, or potentially rename it with a .bak
if run_file_path is not None and cleanup:
try:
parent_dir = os.path.dirname(run_file_path)
if os.path.isdir(parent_dir):
shutil.rmtree(parent_dir)
except Exception, e:
Logger.error(
"Could not cleanup LLAP app package. Error: " + str(e))
# throw the original exception
raise
def _prepare_tez_tarball():
"""
Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
It's very important to use the version of mapreduce which matches tez here.
Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the
GPL license has been accepted.
:return: the full path of the newly created tez tarball to use
"""
import tempfile
Logger.info("Preparing the Tez tarball...")
# get the mapreduce tarball which matches the version of tez
# tez installs the mapreduce tar, so it should always be present
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
_, tez_source_file, _, _ = get_tarball_paths("tez")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
tez_temp_dir = tempfile.mkdtemp(prefix="tez-tarball-", dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
sudo.chmod(tez_temp_dir, 0777)
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir)
Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir))
tar_archive.untar_archive(tez_source_file, tez_temp_dir)
hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib",
"native")
tez_lib_dir = os.path.join(tez_temp_dir, "lib")
if not os.path.exists(hadoop_lib_native_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_dir))
if not os.path.exists(tez_lib_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist"
.format(tez_lib_dir))
# copy native libraries from hadoop to tez
Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo=True)
# if enabled, LZO GPL libraries must be copied as well
if lzo_utils.should_install_lzo():
stack_root = Script.get_stack_root()
service_version = component_version.get_component_repository_version(
service_name="TEZ")
# some installations might not have Tez, but MapReduce2 should be a fallback to get the LZO libraries from
if service_version is None:
Logger.warning(
"Tez does not appear to be installed, using the MapReduce version to get the LZO libraries"
)
service_version = component_version.get_component_repository_version(
service_name="MAPREDUCE2")
hadoop_lib_native_lzo_dir = os.path.join(stack_root, service_version,
"hadoop", "lib", "native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
Logger.warning(
"Unable to located native LZO libraries at {0}, falling back to hadoop home"
.format(hadoop_lib_native_lzo_dir))
hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current",
"hadoop-client", "lib",
"native")
if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
raise Fail(
"Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}"
.format(hadoop_lib_native_lzo_dir))
Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir),
sudo=True)
# ensure that the tez/lib directory is readable by non-root (which it typically is not)
Directory(tez_lib_dir, mode=0755, cd_access='a', recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
tez_native_tarball_staging_dir = os.path.join(
temp_dir, "tez-native-tarball-staging")
if not os.path.exists(tez_native_tarball_staging_dir):
Directory(tez_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
tez_tarball_with_native_lib = os.path.join(tez_native_tarball_staging_dir,
"tez-native.tar.gz")
Logger.info("Creating a new Tez tarball at {0}".format(
tez_tarball_with_native_lib))
tar_archive.archive_dir_via_temp_file(tez_tarball_with_native_lib,
tez_temp_dir)
# ensure that the tarball can be read and uploaded
sudo.chmod(tez_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
sudo.rmtree(tez_temp_dir)
return tez_tarball_with_native_lib
def _prepare_mapreduce_tarball():
"""
Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO is
not enabled or has not been opted-in, then this will do nothing and return the original
tarball to upload to HDFS.
:return: the full path of the newly created mapreduce tarball to use or the original path
if no changes were made
"""
# get the mapreduce tarball to crack open and add LZO libraries to
_, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
if not lzo_utils.should_install_lzo():
return mapreduce_source_file
Logger.info("Preparing the mapreduce tarball with native LZO libraries...")
temp_dir = Script.get_tmp_dir()
# create the temp staging directories ensuring that non-root agents using tarfile can work with them
mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-",
dir=temp_dir)
sudo.chmod(mapreduce_temp_dir, 0777)
# calculate the source directory for LZO
hadoop_lib_native_source_dir = os.path.join(
os.path.dirname(mapreduce_source_file), "lib", "native")
if not sudo.path_exists(hadoop_lib_native_source_dir):
raise Fail(
"Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist"
.format(hadoop_lib_native_source_dir))
Logger.info("Extracting {0} to {1}".format(mapreduce_source_file,
mapreduce_temp_dir))
tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir)
mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib")
# copy native libraries from source hadoop to target
Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir),
sudo=True)
# ensure that the hadoop/lib/native directory is readable by non-root (which it typically is not)
Directory(mapreduce_lib_dir,
mode=0755,
cd_access='a',
recursive_ownership=True)
# create the staging directory so that non-root agents can write to it
mapreduce_native_tarball_staging_dir = os.path.join(
temp_dir, "mapreduce-native-tarball-staging")
if not os.path.exists(mapreduce_native_tarball_staging_dir):
Directory(mapreduce_native_tarball_staging_dir,
mode=0777,
cd_access='a',
create_parents=True,
recursive_ownership=True)
mapreduce_tarball_with_native_lib = os.path.join(
mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz")
Logger.info("Creating a new mapreduce tarball at {0}".format(
mapreduce_tarball_with_native_lib))
tar_archive.archive_dir_via_temp_file(mapreduce_tarball_with_native_lib,
mapreduce_temp_dir)
# ensure that the tarball can be read and uploaded
sudo.chmod(mapreduce_tarball_with_native_lib, 0744)
# cleanup
sudo.rmtree(mapreduce_temp_dir)
return mapreduce_tarball_with_native_lib
