def select_all(version_to_select): """ Executes <stack-selector-tool> on every component for the specified version. If the value passed in is a stack version such as "2.3", then this will find the latest installed version which could be "2.3.0.0-9999". If a version is specified instead, such as 2.3.0.0-1234, it will use that exact version. :param version_to_select: the version to <stack-selector-tool> on, such as "2.3" or "2.3.0.0-1234" """ stack_root = Script.get_stack_root() (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool( stack_tools.STACK_SELECTOR_NAME) # it's an error, but it shouldn't really stop anything from working if version_to_select is None: Logger.error( format( "Unable to execute {stack_selector_name} after installing because there was no version specified" )) return Logger.info("Executing {0} set all on {1}".format(stack_selector_name, version_to_select)) command = format( '{sudo} {stack_selector_path} set all `ambari-python-wrap {stack_selector_path} versions | grep ^{version_to_select} | tail -1`' ) only_if_command = format('ls -d {stack_root}/{version_to_select}*') Execute(command, only_if=only_if_command)
def get_component_version(stack_name, component_name): """ For any stack name, returns the version currently installed for a given component. Because each stack name may have different logic, the input is a generic dictionary. :param stack_name: one of HDP, HDPWIN, BIGTOP, PHD, etc. usually retrieved from the command-#.json file's ["hostLevelParams"]["stack_name"] :param component_name: Component name as a string necessary to get the version :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None """ version = None if stack_name is None or component_name is None: Logger.error("Could not determine component version because of the parameters is empty. " \ "stack_name: %s, component_name: %s" % (str(stack_name), str(component_name))) return version out = None code = -1 if not stack_name: Logger.error("Stack name not provided") elif not component_name: Logger.error("Component name not provided") else: (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool( stack_tools.STACK_SELECTOR_NAME) if stack_selector_name and stack_selector_path and os.path.exists( stack_selector_path): tmpfile = tempfile.NamedTemporaryFile() get_stack_comp_version_cmd = "" try: # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088 with open(tmpfile.name, 'r') as file: get_stack_comp_version_cmd = '%s status %s > %s' % ( stack_selector_path, component_name, tmpfile.name) code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True) out = file.read() if code != 0 or out is None: raise Exception("Code is nonzero or output is empty") Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out))) matches = re.findall(r"( [\d\.]+(\-\d+)?)", out) version = matches[0][0].strip() if matches and len( matches) > 0 and len(matches[0]) > 0 else None Logger.debug("Version for component %s: %s" % (component_name, str(version))) except Exception, e: Logger.error( "Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." % (component_name, get_stack_comp_version_cmd, str(code), str(out))) else:
def get_component_version_from_symlink(stack_name, component_name): """ Gets the version of the specified component by invoking the stack-select tool to query for the version which is referenced by the symlink. :param stack_name: one of HDP, HDPWIN, BIGTOP, PHD, etc. usually retrieved from the command-#.json file's ["clusterLevelParams"]["stack_name"] :param component_name: Component name as a string necessary to get the version :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None """ version = None if not stack_name or not component_name: Logger.error("Could not determine component version because of the parameters is empty. " \ "stack_name: %s, component_name: %s" % (str(stack_name), str(component_name))) return version stack_selector_name, stack_selector_path, stack_selector_package = stack_tools.get_stack_tool( stack_tools.STACK_SELECTOR_NAME) if stack_selector_name and stack_selector_path and os.path.exists( stack_selector_path): tmpfile = tempfile.NamedTemporaryFile(delete=True) out, code = None, -1 get_stack_comp_version_cmd = '%s status %s > %s' % ( stack_selector_path, component_name, tmpfile.name) try: # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088 # ToDo: original problem looks strange with open(tmpfile.name, 'r') as f: code = shell.call(get_stack_comp_version_cmd, quiet=True)[0] out = f.read() if code != 0 or out is None: raise ValueError("Code is nonzero or output is empty") Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out))) matches = re.findall(r"( [\d\.]+(\-\d+)?)", out) version = matches[0][0].strip( ) if matches and len(matches) > 0 and len(matches[0]) > 0 else None Logger.debug("Version for component %s: %s" % (component_name, str(version))) except Exception, e: Logger.error( "Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." % (component_name, get_stack_comp_version_cmd, str(code), str(out)))
def install(self, env): print "Install" self.prepare() component_name = self.get_component_name() repo_info = str(default("/hostLevelParams/repo_info", "1.1.1.1-1")) matches = re.findall(r"([\d\.]+\-\d+)", repo_info) version = matches[0] if matches and len(matches) > 0 else "1.1.1.1-1" from resource_management.libraries.functions import stack_tools (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool( stack_tools.STACK_SELECTOR_NAME) command = 'ambari-python-wrap {0} install {1}'.format( stack_selector_path, version) Execute(command) if component_name: conf_select.select("PERF", component_name, version) stack_select.select(component_name, version)
def select_all(version_to_select): """ Executes <stack-selector-tool> on every component for the specified version. If the value passed in is a stack version such as "2.3", then this will find the latest installed version which could be "2.3.0.0-9999". If a version is specified instead, such as 2.3.0.0-1234, it will use that exact version. :param version_to_select: the version to <stack-selector-tool> on, such as "2.3" or "2.3.0.0-1234" """ stack_root = Script.get_stack_root() (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME) # it's an error, but it shouldn't really stop anything from working if version_to_select is None: Logger.error(format("Unable to execute {stack_selector_name} after installing because there was no version specified")) return Logger.info("Executing {0} set all on {1}".format(stack_selector_name, version_to_select)) command = format('{sudo} {stack_selector_path} set all `ambari-python-wrap {stack_selector_path} versions | grep ^{version_to_select} | tail -1`') only_if_command = format('ls -d {stack_root}/{version_to_select}*') Execute(command, only_if = only_if_command)
def copy_tarballs_to_hdfs(tarball_prefix, stack_select_component_name, component_user, file_owner, group_owner, ignore_sysprep=False): """ :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig :param stack_select_component_name: Component name to get the status to determine the version :param component_user: User that will execute the Hadoop commands, usually smokeuser :param file_owner: Owner of the files copied to HDFS (typically hdfs user) :param group_owner: Group owner of the files copied to HDFS (typically hadoop group) :param ignore_sysprep: Ignore sysprep directives :return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception. In order to call this function, params.py must have all of the following, stack_version_formatted, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab, hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function. """ import params if not ignore_sysprep and hasattr( params, "host_sys_prepped") and params.host_sys_prepped: Logger.info( "Host is sys-prepped. Tarball %s will not be copied for %s." % (tarball_prefix, stack_select_component_name)) return 0 if not hasattr(params, "stack_version_formatted" ) or params.stack_version_formatted is None: Logger.warning("Could not find stack_version_formatted") return 1 component_tar_source_file, component_tar_destination_folder = _get_tar_source_and_dest_folder( tarball_prefix) if not component_tar_source_file or not component_tar_destination_folder: Logger.warning( "Could not retrieve properties for tarball with prefix: %s" % str(tarball_prefix)) return 1 if not os.path.exists(component_tar_source_file): Logger.warning("Could not find file: %s" % str(component_tar_source_file)) return 1 # Ubuntu returns: "stdin: is not a tty", as subprocess32 output. tmpfile = tempfile.NamedTemporaryFile() out = None (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool( stack_tools.STACK_SELECTOR_NAME) with open(tmpfile.name, 'r+') as file: get_stack_version_cmd = '%s status %s > %s' % ( stack_selector_path, stack_select_component_name, tmpfile.name) code, stdoutdata = shell.call(get_stack_version_cmd) out = file.read() pass if code != 0 or out is None: Logger.warning( "Could not verify stack version by calling '%s'. Return Code: %s, Output: %s." % (get_stack_version_cmd, str(code), str(out))) return 1 matches = re.findall(r"([\d\.]+(?:-\d+)?)", out) stack_version = matches[0] if matches and len(matches) > 0 else None if not stack_version: Logger.error("Could not parse stack version from output of %s: %s" % (stack_selector_name, str(out))) return 1 file_name = os.path.basename(component_tar_source_file) destination_file = os.path.join(component_tar_destination_folder, file_name) destination_file = destination_file.replace( "{{ stack_version_formatted }}", stack_version) does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file kinit_if_needed = "" if params.security_enabled: kinit_if_needed = format( "{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") if kinit_if_needed: Execute(kinit_if_needed, user=component_user, path='/bin') does_hdfs_file_exist = False try: ExecuteHadoop(does_hdfs_file_exist_cmd, user=component_user, logoutput=True, conf_dir=params.hadoop_conf_dir, bin_dir=params.hadoop_bin_dir) does_hdfs_file_exist = True except Fail: pass if not does_hdfs_file_exist: source_and_dest_pairs = [ (component_tar_source_file, destination_file), ] return _copy_files(source_and_dest_pairs, component_user, file_owner, group_owner, kinit_if_needed) return 1
def copy_tarballs_to_hdfs(tarball_prefix, stack_select_component_name, component_user, file_owner, group_owner, ignore_sysprep=False): """ :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig :param stack_select_component_name: Component name to get the status to determine the version :param component_user: User that will execute the Hadoop commands, usually smokeuser :param file_owner: Owner of the files copied to HDFS (typically hdfs user) :param group_owner: Group owner of the files copied to HDFS (typically hadoop group) :param ignore_sysprep: Ignore sysprep directives :return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception. In order to call this function, params.py must have all of the following, stack_version_formatted, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab, hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function. """ import params if not ignore_sysprep and hasattr(params, "host_sys_prepped") and params.host_sys_prepped: Logger.info("Host is sys-prepped. Tarball %s will not be copied for %s." % (tarball_prefix, stack_select_component_name)) return 0 if not hasattr(params, "stack_version_formatted") or params.stack_version_formatted is None: Logger.warning("Could not find stack_version_formatted") return 1 component_tar_source_file, component_tar_destination_folder = _get_tar_source_and_dest_folder(tarball_prefix) if not component_tar_source_file or not component_tar_destination_folder: Logger.warning("Could not retrieve properties for tarball with prefix: %s" % str(tarball_prefix)) return 1 if not os.path.exists(component_tar_source_file): Logger.warning("Could not find file: %s" % str(component_tar_source_file)) return 1 # Ubuntu returns: "stdin: is not a tty", as subprocess output. tmpfile = tempfile.NamedTemporaryFile() out = None (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME) with open(tmpfile.name, 'r+') as file: get_stack_version_cmd = '%s status %s > %s' % (stack_selector_path, stack_select_component_name, tmpfile.name) code, stdoutdata = shell.call(get_stack_version_cmd) out = file.read() pass if code != 0 or out is None: Logger.warning("Could not verify stack version by calling '%s'. Return Code: %s, Output: %s." % (get_stack_version_cmd, str(code), str(out))) return 1 matches = re.findall(r"([\d\.]+\-\d+)", out) stack_version = matches[0] if matches and len(matches) > 0 else None if not stack_version: Logger.error("Could not parse stack version from output of %s: %s" % (stack_selector_name, str(out))) return 1 file_name = os.path.basename(component_tar_source_file) destination_file = os.path.join(component_tar_destination_folder, file_name) destination_file = destination_file.replace("{{ stack_version_formatted }}", stack_version) does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file kinit_if_needed = "" if params.security_enabled: kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") if kinit_if_needed: Execute(kinit_if_needed, user=component_user, path='/bin' ) does_hdfs_file_exist = False try: ExecuteHadoop(does_hdfs_file_exist_cmd, user=component_user, logoutput=True, conf_dir=params.hadoop_conf_dir, bin_dir=params.hadoop_bin_dir ) does_hdfs_file_exist = True except Fail: pass if not does_hdfs_file_exist: source_and_dest_pairs = [(component_tar_source_file, destination_file), ] return _copy_files(source_and_dest_pairs, component_user, file_owner, group_owner, kinit_if_needed) return 1