def _update_repo_metadata_after_bad_try(self, cmd, code, out): name = self.get_package_name_with_version() repo_update_cmd = self.get_repo_update_cmd() Logger.info("Execution of '%s' returned %d. %s" % (shell.string_cmd_from_args_list(cmd), code, out)) Logger.info("Failed to install package %s. Executing '%s'" % (name, shell.string_cmd_from_args_list(repo_update_cmd))) code, out = shell.call(repo_update_cmd, sudo=True, logoutput=self.get_logoutput()) if code: Logger.info("Execution of '%s' returned %d. %s" % (repo_update_cmd, code, out)) Logger.info("Retrying to install package %s after %d seconds" % (name, self.resource.retry_sleep))
def remove_package(self, name): if self._check_existence(name): cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self, name, use_repos={}, skip_repos=[], is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] use_repos = use_repos.keys() if use_repos: active_base_repos = self.get_active_base_repos() if 'base' in use_repos: # Remove 'base' from use_repos list use_repos = filter(lambda x: x != 'base', use_repos) use_repos.extend(active_base_repos) use_repos_options = [] for repo in sorted(use_repos): use_repos_options = use_repos_options + ['--repo', repo] cmd = cmd + use_repos_options cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installation of existing package %s" % (name))
def get_user_call_output(command, user, is_checked_call=True, **call_kwargs): """ This function eliminates only output of command inside the su, ignoring the su ouput itself. This is useful since some users have motd messages setup by default on su -l. @return: code, stdout, stderr """ command_string = shell.string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command out_files = [] try: out_files.append(tempfile.NamedTemporaryFile()) out_files.append(tempfile.NamedTemporaryFile()) # other user should be able to write to it for f in out_files: os.chmod(f.name, 0666) command_string += " 1>" + out_files[0].name command_string += " 2>" + out_files[1].name func = shell.checked_call if is_checked_call else shell.call func_result = func(shell.as_user(command_string, user), **call_kwargs) files_output = [] for f in out_files: files_output.append(f.read()) return func_result[0], files_output[0], files_output[1] finally: for f in out_files: f.close()
def gpsshify(command, host=None, hostfile=None, args=None): """Return a gpssh command which will run the command on the specified remote machine in the cluster. NOTE: User must be specified when running command using Execute! Even if user is root. command -- Command or commands which will be run on remote machines in the cluster. host -- A remote host to run the command on. hostfile -- The path to a file containing a list of hosts to run the command on. args -- Additional arguments to append to the gpssh command. """ if host == None and hostfile == None: raise ValueError('Either host or hostfile must be given') if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command) arguments = [] if host != None: arguments.append('-h "%s"' % host) if hostfile != None: arguments.append('-f "%s"' % hostfile) if args != None: arguments.append(args) arguments = " ".join(arguments) return format( dedent(""" cat <<EOF | gpssh {arguments} {command} EOF """.rstrip()))
def run_command(self, target, operation, method='POST', assertable_result=True, file_to_put=None, ignore_status_codes=[], **kwargs): """ assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}' depending on if query was successful or not, we can assert this for them """ target = HdfsResourceProvider.parse_path(target) url = format( "{address}/webhdfs/v1{target}?op={operation}&user.name={run_user}", address=self.address, run_user=self.run_user) for k, v in kwargs.iteritems(): url = format("{url}&{k}={v}") if file_to_put and not os.path.exists(file_to_put): raise Fail(format("File {file_to_put} is not found.")) cmd = ["curl", "-sS", "-L", "-w", "%{http_code}", "-X", method] if file_to_put: cmd += [ "--data-binary", "@" + file_to_put, "-H", "Content-Type: application/octet-stream" ] if self.security_enabled: cmd += ["--negotiate", "-u", ":"] if self.is_https_enabled: cmd += ["-k"] cmd.append(url) _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False) status_code = out[-3:] out = out[:-3] # remove last line from output which is status code try: result_dict = json.loads(out) except ValueError: result_dict = out if status_code not in WebHDFSUtil.valid_status_codes + ignore_status_codes or assertable_result and result_dict and not result_dict[ 'boolean']: formatted_output = json.dumps(result_dict, indent=2) if isinstance( result_dict, dict) else result_dict formatted_output = err + "\n" + formatted_output err_msg = "Execution of '%s' returned status_code=%s. %s" % ( shell.string_cmd_from_args_list(cmd), status_code, formatted_output) raise Fail(err_msg) return result_dict
def remove_package(self, name): if self._check_existence(name): cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] copied_sources_files = [] is_tmp_dir_created = False if use_repos: is_tmp_dir_created = True apt_sources_list_tmp_dir = tempfile.mkdtemp(suffix="-ambari-apt-sources-d") Logger.info("Temporal sources directory was created: %s" % apt_sources_list_tmp_dir) if 'base' not in use_repos: cmd = cmd + ['-o', 'Dir::Etc::SourceList=%s' % EMPTY_FILE] for repo in use_repos: if repo != 'base': new_sources_file = os.path.join(apt_sources_list_tmp_dir, repo + '.list') Logger.info("Temporal sources file will be copied: %s" % new_sources_file) sudo.copy(os.path.join(APT_SOURCES_LIST_DIR, repo + '.list'), new_sources_file) copied_sources_files.append(new_sources_file) cmd = cmd + ['-o', 'Dir::Etc::SourceParts=%s' % apt_sources_list_tmp_dir] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) code, out = self.call_with_retries(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if self.is_locked_output(out): err_msg = Logger.filter_text("Execution of '%s' returned %d. %s" % (cmd, code, out)) raise Fail(err_msg) # apt-get update wasn't done too long maybe? if code: Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out)) Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD))) code, out = self.call_with_retries(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput()) if code: Logger.info("Execution of '%s' returned %d. %s" % (REPO_UPDATE_CMD, code, out)) Logger.info("Retrying to install package %s" % (name)) self.checked_call_with_retries(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if is_tmp_dir_created: for temporal_sources_file in copied_sources_files: Logger.info("Removing temporal sources file: %s" % temporal_sources_file) os.remove(temporal_sources_file) Logger.info("Removing temporal sources directory: %s" % apt_sources_list_tmp_dir) os.rmdir(apt_sources_list_tmp_dir) else: Logger.info("Skipping installation of existing package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] copied_sources_files = [] is_tmp_dir_created = False if use_repos: is_tmp_dir_created = True apt_sources_list_tmp_dir = tempfile.mkdtemp(suffix="-ambari-apt-sources-d") Logger.info("Temporal sources directory was created: %s" % apt_sources_list_tmp_dir) if 'base' not in use_repos: cmd = cmd + ['-o', 'Dir::Etc::SourceList=%s' % EMPTY_FILE] for repo in use_repos: if repo != 'base': new_sources_file = os.path.join(apt_sources_list_tmp_dir, repo + '.list') Logger.info("Temporal sources file will be copied: %s" % new_sources_file) sudo.copy(os.path.join(APT_SOURCES_LIST_DIR, repo + '.list'), new_sources_file) copied_sources_files.append(new_sources_file) cmd = cmd + ['-o', 'Dir::Etc::SourceParts=%s' % apt_sources_list_tmp_dir] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) code, out = self.call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if self.is_locked_output(out): err_msg = Logger.filter_text("Execution of '%s' returned %d. %s" % (cmd, code, out)) raise Fail(err_msg) # apt-get update wasn't done too long maybe? if code: Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out)) Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD))) code, out = self.call_until_not_locked(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput()) if code: Logger.info("Execution of '%s' returned %d. %s" % (REPO_UPDATE_CMD, code, out)) Logger.info("Retrying to install package %s" % (name)) self.checked_call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if is_tmp_dir_created: for temporal_sources_file in copied_sources_files: Logger.info("Removing temporal sources file: %s" % temporal_sources_file) os.remove(temporal_sources_file) Logger.info("Removing temporal sources directory: %s" % apt_sources_list_tmp_dir) os.rmdir(apt_sources_list_tmp_dir) else: Logger.info("Skipping installation of existing package %s" % (name))
def install_package(self, name, use_repos={}, skip_repos=set(), is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] copied_sources_files = [] is_tmp_dir_created = False if use_repos: if 'base' in use_repos: use_repos = set( [v for k, v in use_repos.items() if k != 'base']) else: cmd = cmd + ['-o', 'Dir::Etc::SourceList=%s' % EMPTY_FILE] use_repos = set(use_repos.values()) if use_repos: is_tmp_dir_created = True apt_sources_list_tmp_dir = tempfile.mkdtemp( suffix="-ambari-apt-sources-d") Logger.info("Temporary sources directory was created: %s" % apt_sources_list_tmp_dir) for repo in use_repos: new_sources_file = os.path.join( apt_sources_list_tmp_dir, repo + '.list') Logger.info( "Temporary sources file will be copied: %s" % new_sources_file) sudo.copy( os.path.join(APT_SOURCES_LIST_DIR, repo + '.list'), new_sources_file) copied_sources_files.append(new_sources_file) cmd = cmd + [ '-o', 'Dir::Etc::SourceParts=%s' % apt_sources_list_tmp_dir ] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput()) if is_tmp_dir_created: for temporary_sources_file in copied_sources_files: Logger.info("Removing temporary sources file: %s" % temporary_sources_file) os.remove(temporary_sources_file) Logger.info("Removing temporary sources directory: %s" % apt_sources_list_tmp_dir) os.rmdir(apt_sources_list_tmp_dir) else: Logger.info("Skipping installation of existing package %s" % (name))
def get_user_call_output(command, user, quiet=False, is_checked_call=True, **call_kwargs): """ This function eliminates only output of command inside the su, ignoring the su ouput itself. This is useful since some users have motd messages setup by default on su -l. @return: code, stdout, stderr """ command_string = shell.string_cmd_from_args_list(command) if isinstance( command, (list, tuple)) else command out_files = [] try: out_files.append(tempfile.NamedTemporaryFile()) out_files.append(tempfile.NamedTemporaryFile()) # other user should be able to write to it for f in out_files: os.chmod(f.name, 0666) command_string += " 1>" + out_files[0].name command_string += " 2>" + out_files[1].name code, _ = shell.call(shell.as_user(command_string, user), quiet=quiet, **call_kwargs) files_output = [] for f in out_files: files_output.append(f.read().decode("utf-8").strip('\n')) if code: all_output = files_output[1] + '\n' + files_output[0] err_msg = Logger.filter_text( ("Execution of '%s' returned %d. %s") % (command_string, code, all_output)) if is_checked_call: raise Fail(err_msg) else: Logger.warning(err_msg) result = code, files_output[0], files_output[1] caller_filename = sys._getframe(1).f_code.co_filename is_internal_call = shell.NOT_LOGGED_FOLDER in caller_filename if quiet == False or (quiet == None and not is_internal_call): log_msg = "{0} returned {1}".format(get_user_call_output.__name__, result) Logger.info(log_msg) return result finally: for f in out_files: f.close()
def remove_package(self, name, ignore_dependencies=False): if self._check_existence(name): if ignore_dependencies: cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [name] else: cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self, name, use_repos=[]): if not self._check_existence(name) or use_repos: cmd = INSTALL_CMD[self.get_logoutput()] if use_repos: enable_repo_option = '--enablerepo=' + ",".join(use_repos) cmd = cmd + ['--disablerepo=*', enable_repo_option] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installing existent package %s" % (name))
def install_package(self, name, use_repos=[]): if not self._check_existence(name) or use_repos: cmd = INSTALL_CMD[self.get_logoutput()] if use_repos: enable_repo_option = "--enablerepo=" + ",".join(use_repos) cmd = cmd + ["--disablerepo=*", enable_repo_option] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installing existent package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] if use_repos: enable_repo_option = '--enablerepo=' + ",".join(use_repos) disable_repo_option = '--disablerepo=' + "*,".join(skip_repos) cmd = cmd + [disable_repo_option, enable_repo_option] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installation of existing package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] if use_repos: enable_repo_option = '--enablerepo=' + ",".join(use_repos) disable_repo_option = '--disablerepo=' + "*,".join(skip_repos) cmd = cmd + [disable_repo_option, enable_repo_option] cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installation of existing package %s" % (name))
def get_user_call_output(command, user, quiet=False, is_checked_call=True, **call_kwargs): """ This function eliminates only output of command inside the su, ignoring the su ouput itself. This is useful since some users have motd messages setup by default on su -l. @return: code, stdout, stderr """ command_string = shell.string_cmd_from_args_list(command) if isinstance( command, (list, tuple)) else command out_files = [] try: out_files.append(tempfile.NamedTemporaryFile()) out_files.append(tempfile.NamedTemporaryFile()) # other user should be able to write to it for f in out_files: os.chmod(f.name, 0666) command_string += " 1>" + out_files[0].name command_string += " 2>" + out_files[1].name code, _ = shell.call(shell.as_user(command_string, user), quiet=quiet, **call_kwargs) files_output = [] for f in out_files: files_output.append(f.read().strip('\n')) if code: all_output = files_output[1] + '\n' + files_output[0] err_msg = Logger.filter_text( ("Execution of '%s' returned %d. %s") % (command_string, code, all_output)) if is_checked_call: raise Fail(err_msg) else: Logger.warning(err_msg) return code, files_output[0], files_output[1] finally: for f in out_files: f.close()
def remove_package(self, name, ignore_dependencies=False): if self._check_existence(name): if ignore_dependencies: cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [ name, name ] # have to specify name twice: one for --ignore-depends, one for --remove else: cmd = REMOVE_CMD[self.get_logoutput()] + [name] Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self, name, use_repos=[]): if not self._check_existence(name) or use_repos: cmd = INSTALL_CMD[self.get_logoutput()] if use_repos: active_base_repos = get_active_base_repos() if 'base' in use_repos: use_repos = filter(lambda x: x != 'base', use_repos) use_repos.extend(active_base_repos) use_repos_options = [] for repo in use_repos: use_repos_options = use_repos_options + ['--repo', repo] cmd = cmd + use_repos_options cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installing existent package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False): if is_upgrade or use_repos or not self._check_existence(name): cmd = INSTALL_CMD[self.get_logoutput()] if use_repos: active_base_repos = self.get_active_base_repos() if 'base' in use_repos: # Remove 'base' from use_repos list use_repos = filter(lambda x: x != 'base', use_repos) use_repos.extend(active_base_repos) use_repos_options = [] for repo in use_repos: use_repos_options = use_repos_options + ['--repo', repo] cmd = cmd + use_repos_options cmd = cmd + [name] Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd))) self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput()) else: Logger.info("Skipping installation of existing package %s" % (name))
def get_user_call_output(command, user, quiet=False, is_checked_call=True, **call_kwargs): """ This function eliminates only output of command inside the su, ignoring the su ouput itself. This is useful since some users have motd messages setup by default on su -l. @return: code, stdout, stderr """ command_string = shell.string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command out_files = [] try: out_files.append(tempfile.NamedTemporaryFile()) out_files.append(tempfile.NamedTemporaryFile()) # other user should be able to write to it for f in out_files: os.chmod(f.name, 0666) command_string += " 1>" + out_files[0].name command_string += " 2>" + out_files[1].name code, _ = shell.call(shell.as_user(command_string, user), quiet=quiet, **call_kwargs) files_output = [] for f in out_files: files_output.append(f.read().strip('\n')) if code: all_output = files_output[1] + '\n' + files_output[0] err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_string, code, all_output)) if is_checked_call: raise Fail(err_msg) else: Logger.warning(err_msg) return code, files_output[0], files_output[1] finally: for f in out_files: f.close()
def run_command(self, target, operation, method='POST', assertable_result=True, file_to_put=None, ignore_status_codes=[], **kwargs): """ assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}' depending on if query was successful or not, we can assert this for them """ target = HdfsResourceProvider.parse_path(target) url = format("{address}/webhdfs/v1{target}?op={operation}&user.name={run_user}", address=self.address, run_user=self.run_user) for k,v in kwargs.iteritems(): url = format("{url}&{k}={v}") if file_to_put and not os.path.exists(file_to_put): raise Fail(format("File {file_to_put} is not found.")) cmd = ["curl", "-sS","-L", "-w", "%{http_code}", "-X", method] if file_to_put: cmd += ["-T", file_to_put] if self.security_enabled: cmd += ["--negotiate", "-u", ":"] if self.is_https_enabled: cmd += ["-k"] cmd.append(url) _, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False) status_code = out[-3:] out = out[:-3] # remove last line from output which is status code try: result_dict = json.loads(out) except ValueError: result_dict = out if status_code not in WebHDFSUtil.valid_status_codes+ignore_status_codes or assertable_result and result_dict and not result_dict['boolean']: formatted_output = json.dumps(result_dict, indent=2) if isinstance(result_dict, dict) else result_dict formatted_output = err + "\n" + formatted_output err_msg = "Execution of '%s' returned status_code=%s. %s" % (shell.string_cmd_from_args_list(cmd), status_code, formatted_output) raise Fail(err_msg) return result_dict