def _get_single_version_from_hdp_select(): """ Call "hdp-select versions" and return the version string if only one version is available. :return: Returns a version string if successful, and None otherwise. """ # Ubuntu returns: "stdin: is not a tty", as subprocess output, so must use a temporary file to store the output. tmpfile = tempfile.NamedTemporaryFile() tmp_dir = Script.get_tmp_dir() tmp_file = os.path.join(tmp_dir, "copy_tarball_out.txt") hdp_version = None out = None get_hdp_versions_cmd = "/usr/bin/hdp-select versions > {0}".format(tmp_file) try: code, stdoutdata = shell.call(get_hdp_versions_cmd, logoutput=True) with open(tmp_file, 'r+') as file: out = file.read() except Exception, e: Logger.logger.exception("Could not parse output of {0}. Error: {1}".format(str(tmp_file), str(e)))
limitations under the License. Ambari Agent """ import status_params from resource_management.libraries.functions import format from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script # server configurations config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) stack_name = default("/hostLevelParams/stack_name", None) current_version = default("/hostLevelParams/current_version", None) component_directory = status_params.component_directory # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) # default parameters zk_home = "/usr/hdp/2.3.2.0-2950/zookeeper" zk_bin = "/usr/hdp/2.3.2.0-2950/zookeeper/bin" zk_cli_shell = "/usr/hdp/2.3.2.0-2950/zookeeper/bin/zkCli.sh"
from resource_management import * from resource_management.libraries.script.script import Script import sys, os, glob, socket # server configurations config = Script.get_config() service_packagedir = os.path.realpath(__file__).split('/scripts')[0] tmp_dir = Script.get_tmp_dir() kylin_properties = config['configurations']['kylin']['kylin_properties'] hdp_version = default("/commandParams/version", None) downloadlocation = "http://mirror.bit.edu.cn/apache/kylin/apache-kylin-2.6.3/apache-kylin-2.6.3-bin-hadoop3.tar.gz" #install_dir = config['configurations']['kylin']['install.dir'] install_dir = format('/opt/kylin') current_host_name = socket.gethostname() server_mode = "query" server_masters = config['clusterHostInfo']['kylin_all_hosts'][0] server_clusters_arr = config['clusterHostInfo']['kylin_all_hosts'] + ( config['clusterHostInfo'].has_key('kylin_query_hosts') and config['clusterHostInfo']['kylin_query_hosts'] or []) server_port = "7070" server_clusters = ','.join(i + ":" + server_port for i in server_clusters_arr) kylin_servers = ';'.join("server " + i + ":" + server_port for i in server_clusters_arr) + ";" hadoop_conf_dir = install_dir + "/latest/conf/hadoop_conf" nginx_conf = config['configurations']['nginx']['nginx_conf'] nginx_port = config['configurations']['nginx']['nginx_port']
def _llap_start(self, env, cleanup=False): import params env.set_params(params) if params.hive_server_interactive_ha: """ Check llap app state """ Logger.info("HSI HA is enabled. Checking if LLAP is already running ...") if params.stack_supports_hive_interactive_ga: status = self.check_llap_app_status_in_llap_ga(params.llap_app_name, 2, params.hive_server_interactive_ha) else: status = self.check_llap_app_status_in_llap_tp(params.llap_app_name, 2, params.hive_server_interactive_ha) if status: Logger.info("LLAP app '{0}' is already running.".format(params.llap_app_name)) return True else: Logger.info("LLAP app '{0}' is not running. llap will be started.".format(params.llap_app_name)) pass # Call for cleaning up the earlier run(s) LLAP package folders. self._cleanup_past_llap_package_dirs() Logger.info("Starting LLAP") LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir() unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --slider-am-container-mb {params.slider_am_container_mb} " "--size {params.llap_daemon_container_size}m --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m " "--loglevel {params.llap_log_level} {params.llap_extra_slider_opts} --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") # Append params that are supported from Hive llap GA version. if params.stack_supports_hive_interactive_ga: # Figure out the Slider Anti-affinity to be used. # YARN does not support anti-affinity, and therefore Slider implements AA by the means of exclusion lists, i.e, it # starts containers one by one and excludes the nodes it gets (adding a delay of ~2sec./machine). When the LLAP # container memory size configuration is more than half of YARN node memory, AA is implicit and should be avoided. slider_placement = 4 if long(params.llap_daemon_container_size) > (0.5 * long(params.yarn_nm_mem)): slider_placement = 0 Logger.info("Setting slider_placement : 0, as llap_daemon_container_size : {0} > 0.5 * " "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) else: Logger.info("Setting slider_placement: 4, as llap_daemon_container_size : {0} <= 0.5 * " "YARN NodeManager Memory({1})".format(params.llap_daemon_container_size, params.yarn_nm_mem)) cmd += format(" --slider-placement {slider_placement} --skiphadoopversion --skiphbasecp --instances {params.num_llap_daemon_running_nodes}") # Setup the logger for the ga version only cmd += format(" --logger {params.llap_logger}") else: cmd += format(" --instances {params.num_llap_nodes}") if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) cmd += format(" --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab " "{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}") # Add the aux jars if they are specified. If empty, dont need to add this param. if params.hive_aux_jars: cmd+= format(" --auxjars {params.hive_aux_jars}") # Append args. llap_java_args = InlineTemplate(params.llap_app_java_opts).get_content() cmd += format(" --args \" {llap_java_args}\"") # Append metaspace size to args. if params.java_version > 7 and params.llap_daemon_container_size > 4096: if params.llap_daemon_container_size <= 32768: metaspaceSize = "256m" else: metaspaceSize = "1024m" cmd = cmd[:-1] + " -XX:MetaspaceSize="+metaspaceSize+ "\"" run_file_path = None try: Logger.info(format("LLAP start command: {cmd}")) code, output, error = shell.checked_call(cmd, user=params.hive_user, quiet = True, stderr=subprocess.PIPE, logoutput=True) if code != 0 or output is None: raise Fail("Command failed with either non-zero return code or no output.") # E.g., output: # Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider exp = r"Prepared (.*?run.sh) for running LLAP" run_file_path = None out_splits = output.split("\n") for line in out_splits: line = line.strip() m = re.match(exp, line, re.I) if m and len(m.groups()) == 1: run_file_name = m.group(1) run_file_path = os.path.join(params.hive_user_home_dir, run_file_name) break if not run_file_path: raise Fail("Did not find run.sh file in output: " + str(output)) Logger.info(format("Run file path: {run_file_path}")) Execute(run_file_path, user=params.hive_user, logoutput=True) Logger.info("Submitted LLAP app name : {0}".format(params.llap_app_name)) # We need to check the status of LLAP app to figure out it got # launched properly and is in running state. Then go ahead with Hive Interactive Server start. if params.stack_supports_hive_interactive_ga: status = self.check_llap_app_status_in_llap_ga(params.llap_app_name, params.num_retries_for_checking_llap_status) else: status = self.check_llap_app_status_in_llap_tp(params.llap_app_name, params.num_retries_for_checking_llap_status) if status: Logger.info("LLAP app '{0}' deployed successfully.".format(params.llap_app_name)) return True else: Logger.error("LLAP app '{0}' deployment unsuccessful.".format(params.llap_app_name)) return False except: # Attempt to clean up the packaged application, or potentially rename it with a .bak if run_file_path is not None and cleanup: parent_dir = os.path.dirname(run_file_path) Directory(parent_dir, action = "delete", ignore_failures = True, ) # throw the original exception raise
def _llap_start(self, env, cleanup=False): import params env.set_params(params) Logger.info("Starting LLAP") LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir() LLAP_APP_NAME = 'llap0' unique_name = "llap-slider%s" % datetime.utcnow().strftime( '%Y-%m-%d_%H-%M-%S') cmd = format( "{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}" " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m " " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}" " --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) cmd += format( " --slider-keytab-dir .slider/keytabs/{params.hive_user}/ --slider-keytab " "{llap_keytab_splits[4]} --slider-principal {params.hive_llap_principal}" ) # Add the aux jars if they are specified. If empty, dont need to add this param. if params.hive_aux_jars: cmd += format(" --auxjars {params.hive_aux_jars}") # Append args. llap_java_args = InlineTemplate( params.llap_app_java_opts).get_content() cmd += format(" --args \" {llap_java_args}\"") run_file_path = None try: Logger.info(format("Command: {cmd}")) code, output, error = shell.checked_call(cmd, user=params.hive_user, stderr=subprocess.PIPE, logoutput=True) if code != 0 or output is None: raise Fail( "Command failed with either non-zero return code or no output." ) # E.g., output: # Prepared llap-slider-05Apr2016/run.sh for running LLAP on Slider exp = r"Prepared (.*?run.sh) for running LLAP" run_file_path = None out_splits = output.split("\n") for line in out_splits: line = line.strip() m = re.match(exp, line, re.I) if m and len(m.groups()) == 1: run_file_name = m.group(1) run_file_path = os.path.join(params.hive_user_home_dir, run_file_name) break if not run_file_path: raise Fail("Did not find run.sh file in output: " + str(output)) Logger.info(format("Run file path: {run_file_path}")) Execute(run_file_path, user=params.hive_user, logoutput=True) Logger.info("Submitted LLAP app name : {0}".format(LLAP_APP_NAME)) # We need to check the status of LLAP app to figure out it got # launched properly and is in running state. Then go ahead with Hive Interactive Server start. status = self.check_llap_app_status( LLAP_APP_NAME, params.num_retries_for_checking_llap_status) if status: Logger.info("LLAP app '{0}' deployed successfully.".format( LLAP_APP_NAME)) return True else: Logger.error("LLAP app '{0}' deployment unsuccessful.".format( LLAP_APP_NAME)) return False except: # Attempt to clean up the packaged application, or potentially rename it with a .bak if run_file_path is not None and cleanup: try: parent_dir = os.path.dirname(run_file_path) if os.path.isdir(parent_dir): shutil.rmtree(parent_dir) except Exception, e: Logger.error( "Could not cleanup LLAP app package. Error: " + str(e)) # throw the original exception raise
def _prepare_tez_tarball(): """ Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball. It's very important to use the version of mapreduce which matches tez here. Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the GPL license has been accepted. :return: the full path of the newly created tez tarball to use """ import tempfile Logger.info("Preparing the Tez tarball...") # get the mapreduce tarball which matches the version of tez # tez installs the mapreduce tar, so it should always be present _, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce") _, tez_source_file, _, _ = get_tarball_paths("tez") temp_dir = Script.get_tmp_dir() # create the temp staging directories ensuring that non-root agents using tarfile can work with them mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-", dir=temp_dir) tez_temp_dir = tempfile.mkdtemp(prefix="tez-tarball-", dir=temp_dir) sudo.chmod(mapreduce_temp_dir, 0777) sudo.chmod(tez_temp_dir, 0777) Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir)) tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir) Logger.info("Extracting {0} to {1}".format(tez_source_file, tez_temp_dir)) tar_archive.untar_archive(tez_source_file, tez_temp_dir) hadoop_lib_native_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib", "native") tez_lib_dir = os.path.join(tez_temp_dir, "lib") if not os.path.exists(hadoop_lib_native_dir): raise Fail( "Unable to seed the Tez tarball with native libraries since the source Hadoop native lib directory {0} does not exist" .format(hadoop_lib_native_dir)) if not os.path.exists(tez_lib_dir): raise Fail( "Unable to seed the Tez tarball with native libraries since the target Tez lib directory {0} does not exist" .format(tez_lib_dir)) # copy native libraries from hadoop to tez Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo=True) # if enabled, LZO GPL libraries must be copied as well if lzo_utils.should_install_lzo(): stack_root = Script.get_stack_root() service_version = component_version.get_component_repository_version( service_name="TEZ") # some installations might not have Tez, but MapReduce2 should be a fallback to get the LZO libraries from if service_version is None: Logger.warning( "Tez does not appear to be installed, using the MapReduce version to get the LZO libraries" ) service_version = component_version.get_component_repository_version( service_name="MAPREDUCE2") hadoop_lib_native_lzo_dir = os.path.join(stack_root, service_version, "hadoop", "lib", "native") if not sudo.path_isdir(hadoop_lib_native_lzo_dir): Logger.warning( "Unable to located native LZO libraries at {0}, falling back to hadoop home" .format(hadoop_lib_native_lzo_dir)) hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current", "hadoop-client", "lib", "native") if not sudo.path_isdir(hadoop_lib_native_lzo_dir): raise Fail( "Unable to seed the Tez tarball with native libraries since LZO is enabled but the native LZO libraries could not be found at {0}" .format(hadoop_lib_native_lzo_dir)) Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir), sudo=True) # ensure that the tez/lib directory is readable by non-root (which it typically is not) Directory(tez_lib_dir, mode=0755, cd_access='a', recursive_ownership=True) # create the staging directory so that non-root agents can write to it tez_native_tarball_staging_dir = os.path.join( temp_dir, "tez-native-tarball-staging") if not os.path.exists(tez_native_tarball_staging_dir): Directory(tez_native_tarball_staging_dir, mode=0777, cd_access='a', create_parents=True, recursive_ownership=True) tez_tarball_with_native_lib = os.path.join(tez_native_tarball_staging_dir, "tez-native.tar.gz") Logger.info("Creating a new Tez tarball at {0}".format( tez_tarball_with_native_lib)) tar_archive.archive_dir_via_temp_file(tez_tarball_with_native_lib, tez_temp_dir) # ensure that the tarball can be read and uploaded sudo.chmod(tez_tarball_with_native_lib, 0744) # cleanup sudo.rmtree(mapreduce_temp_dir) sudo.rmtree(tez_temp_dir) return tez_tarball_with_native_lib
def _prepare_mapreduce_tarball(): """ Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO is not enabled or has not been opted-in, then this will do nothing and return the original tarball to upload to HDFS. :return: the full path of the newly created mapreduce tarball to use or the original path if no changes were made """ # get the mapreduce tarball to crack open and add LZO libraries to _, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce") if not lzo_utils.should_install_lzo(): return mapreduce_source_file Logger.info("Preparing the mapreduce tarball with native LZO libraries...") temp_dir = Script.get_tmp_dir() # create the temp staging directories ensuring that non-root agents using tarfile can work with them mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-", dir=temp_dir) sudo.chmod(mapreduce_temp_dir, 0777) # calculate the source directory for LZO hadoop_lib_native_source_dir = os.path.join( os.path.dirname(mapreduce_source_file), "lib", "native") if not sudo.path_exists(hadoop_lib_native_source_dir): raise Fail( "Unable to seed the mapreduce tarball with native LZO libraries since the source Hadoop native lib directory {0} does not exist" .format(hadoop_lib_native_source_dir)) Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir)) tar_archive.untar_archive(mapreduce_source_file, mapreduce_temp_dir) mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib") # copy native libraries from source hadoop to target Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir), sudo=True) # ensure that the hadoop/lib/native directory is readable by non-root (which it typically is not) Directory(mapreduce_lib_dir, mode=0755, cd_access='a', recursive_ownership=True) # create the staging directory so that non-root agents can write to it mapreduce_native_tarball_staging_dir = os.path.join( temp_dir, "mapreduce-native-tarball-staging") if not os.path.exists(mapreduce_native_tarball_staging_dir): Directory(mapreduce_native_tarball_staging_dir, mode=0777, cd_access='a', create_parents=True, recursive_ownership=True) mapreduce_tarball_with_native_lib = os.path.join( mapreduce_native_tarball_staging_dir, "mapreduce-native.tar.gz") Logger.info("Creating a new mapreduce tarball at {0}".format( mapreduce_tarball_with_native_lib)) tar_archive.archive_dir_via_temp_file(mapreduce_tarball_with_native_lib, mapreduce_temp_dir) # ensure that the tarball can be read and uploaded sudo.chmod(mapreduce_tarball_with_native_lib, 0744) # cleanup sudo.rmtree(mapreduce_temp_dir) return mapreduce_tarball_with_native_lib