def _call(command, logoutput=False, throw_on_failure=True, cwd=None, env=None, preexec_fn=None): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @return: retrun_code, stdout """ # convert to string and escape if isinstance(command, (list, tuple)): command = ' '.join(pipes.quote(x) for x in command) command = ["/bin/bash","--login","-c", command] proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) out = proc.communicate()[0] code = proc.returncode if logoutput and out and out!="": log.info(out) if throw_on_failure and code: err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out) raise Fail(err_msg) return code, out
def quote_bash_args(command): if not command: return "''" if not isinstance(command, basestring): raise Fail("Command should be a list of strings, found '{0}' in command list elements".format(str(command))) valid = set(string.ascii_letters + string.digits + '@%_-+=:,./') for char in command: if char not in valid: return "'" + command.replace("'", "'\"'\"'") + "'" return command
def _call(command, logoutput=False, throw_on_failure=True, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @return: retrun_code, stdout """ # convert to string and escape if isinstance(command, (list, tuple)): command = ' '.join(pipes.quote(x) for x in command) """ Do not su to the supplied user (need to differentiate between when to call su and when not to) if user: command = ["su", "-", user, "-c", command] else: command = ["/bin/bash","--login","-c", command] """ command = ["/bin/bash", "--login", "-c", command] proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) if not wait_for_finish: return None, None out = proc.communicate()[0].strip('\n') code = proc.returncode if logoutput and out: Logger.info(out) if throw_on_failure and code: err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out) raise Fail(err_msg) return code, out
def _call(command, logoutput=False, throw_on_failure=True, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @return: retrun_code, stdout """ # convert to string and escape if isinstance(command, (list, tuple)): command = ' '.join(quote_bash_args(x) for x in command) if user: command = ["su", "-", user, "-c", command] else: command = ["/bin/bash","--login","-c", command] proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) if not wait_for_finish: return None, None if timeout: q = Queue() t = threading.Timer( timeout, on_timeout, [proc, q] ) t.start() out = proc.communicate()[0].strip('\n') if timeout: if q.empty(): t.cancel() # timeout occurred else: raise ExecuteTimeoutException() code = proc.returncode if logoutput and out: Logger.info(out) if throw_on_failure and code: err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out) raise Fail(err_msg) return code, out
def as_sudo(command, env=None, auto_escape=True): """ command - list or tuple of arguments. env - when run as part of Execute resource, this SHOULD NOT be used. It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if """ if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command, auto_escape=auto_escape) else: # Since ambari user sudoer privileges may be restricted, # without having /bin/bash permission, and /bin/su permission. # Running interpreted shell commands in scope of 'sudo' is not possible. # # In that case while passing string, # any bash symbols eventually added to command like && || ; < > | << >> would cause problems. err_msg = Logger.filter_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command)) raise Fail(err_msg) env = _get_environment_str(_add_current_path_to_env(env)) if env else ENV_PLACEHOLDER return "{0} {1} -H -E {2}".format(_get_sudo_binary(), env, command)
def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None, path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @param stdout,stderr: subprocess.PIPE - enable output to variable subprocess.STDOUT - redirect to stdout None - disable output to variable, and output to Python out straightly (even if logoutput is False) {int fd} - redirect to file with descriptor. {string filename} - redirects to a file with name. """ command_alias = Logger.format_command_for_output(command) command_alias = string_cmd_from_args_list(command_alias) if isinstance(command_alias, (list, tuple)) else command_alias # Append current PATH to env['PATH'] env = _add_current_path_to_env(env) # Append path to env['PATH'] if path: path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path env['PATH'] = os.pathsep.join([env['PATH'], path]) # prepare command cmd if sudo: command = as_sudo(command, env=env) elif user: command = as_user(command, user, env=env) # convert to string and escape if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command) # replace placeholder from as_sudo / as_user if present env_str = _get_environment_str(env) for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems(): command = command.replace(placeholder, replacement.format(env_str=env_str)) # --noprofile is used to preserve PATH set for ambari-agent subprocess_command = ["/bin/bash","--login","--noprofile","-c", command] files_to_close = [] if isinstance(stdout, (basestring)): stdout = open(stdout, 'wb') files_to_close.append(stdout) if isinstance(stderr, (basestring)): stderr = open(stderr, 'wb') files_to_close.append(stderr) try: proc = subprocess.Popen(subprocess_command, stdout=stdout, stderr=stderr, cwd=cwd, env=env, shell=False, close_fds=True, preexec_fn=preexec_fn) if timeout: timeout_event = threading.Event() t = threading.Timer( timeout, _on_timeout, [proc, timeout_event] ) t.start() if not wait_for_finish: return proc # in case logoutput==False, never log. logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG) read_set = [] if stdout == subprocess.PIPE: read_set.append(proc.stdout) if stderr == subprocess.PIPE: read_set.append(proc.stderr) fd_to_string = { proc.stdout: "", proc.stderr: "" } all_output = "" while read_set: is_proccess_running = (proc.poll() == None) ready, _, _ = select.select(read_set, [], [], 1) if not is_proccess_running and not ready: break for out_fd in read_set: if out_fd in ready: line = os.read(out_fd.fileno(), 1024) if not line: read_set = copy.copy(read_set) read_set.remove(out_fd) out_fd.close() continue fd_to_string[out_fd] += line all_output += line if on_new_line: try: on_new_line(line, out_fd == proc.stderr) except Exception, err: err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(line, traceback.format_exc()) raise Fail(err_msg) if logoutput: _print(line) # Wait for process to terminate if not timeout or not timeout_event.is_set(): proc.wait()
err = fd_to_string[proc.stderr].strip('\n') all_output = all_output.strip('\n') if timeout: if not timeout_event.is_set(): t.cancel() # timeout occurred else: err_msg = Logger.filter_text(("Execution of '%s' was killed due timeout after %d seconds") % (command_alias, timeout)) raise ExecuteTimeoutException(err_msg) code = proc.returncode if throw_on_failure and code: err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias, code, all_output)) raise Fail(err_msg) # if separate stderr is enabled (by default it's redirected to out) if stderr == subprocess.PIPE: return code, out, err return code, out def as_sudo(command, env=None, auto_escape=True): """ command - list or tuple of arguments. env - when run as part of Execute resource, this SHOULD NOT be used. It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if """ if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command, auto_escape=auto_escape)
def _call(command, logoutput=None, throw_on_failure=True, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, path=None, sudo=False, on_new_line=None): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown """ command_alias = string_cmd_from_args_list(command) if isinstance( command, (list, tuple)) else command # Append current PATH to env['PATH'] env = _add_current_path_to_env(env) # Append path to env['PATH'] if path: path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path env['PATH'] = os.pathsep.join([env['PATH'], path]) # prepare command cmd if sudo: command = as_sudo(command, env=env) elif user: command = as_user(command, user, env=env) # convert to string and escape if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command) # replace placeholder from as_sudo / as_user if present env_str = _get_environment_str(env) for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems(): command = command.replace(placeholder, replacement.format(env_str=env_str)) master_fd, slave_fd = pty.openpty() # --noprofile is used to preserve PATH set for ambari-agent subprocess_command = ["/bin/bash", "--login", "--noprofile", "-c", command] proc = subprocess.Popen(subprocess_command, bufsize=1, stdout=slave_fd, stderr=subprocess.STDOUT, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) if timeout: timeout_event = threading.Event() t = threading.Timer(timeout, _on_timeout, [proc, timeout_event]) t.start() if not wait_for_finish: return proc # in case logoutput==False, never log. logoutput = logoutput == True and Logger.logger.isEnabledFor( logging.INFO) or logoutput == None and Logger.logger.isEnabledFor( logging.DEBUG) out = "" read_timeout = .04 # seconds try: while True: ready, _, _ = select.select([master_fd], [], [], read_timeout) if ready: line = os.read(master_fd, 512) if not line: break out += line try: if on_new_line: on_new_line(line) except Exception, err: err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format( line, traceback.format_exc()) raise Fail(err_msg) if logoutput: _print(line) elif proc.poll() is not None: break # proc exited
def _call(command, logoutput=False, throw_on_failure=True, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file_name=None, poll_after=None): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @return: retrun_code, stdout """ # convert to string and escape if isinstance(command, (list, tuple)): command = ' '.join(quote_bash_args(x) for x in command) """ Do not su to the supplied user (need to differentiate between when to call su and when not to) if user: command = ["su", "-", user, "-c", command] else: command = ["/bin/bash","--login","-c", command] """ command = ["/bin/bash", "--login", "-c", command] #adding redirecting stdout stderr to file outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE stdoutFile = open(outfilename, 'w+') stderrFile = open(errfilename, 'w+') proc = subprocess.Popen(command, stdout=stdoutFile, stderr=stderrFile, universal_newlines=True, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) logAnyway = False if not wait_for_finish: if pid_file_name: pidfile = open(pid_file_name, 'w') pidfile.write(str(proc.pid)) pidfile.close() ## wait poll_after seconds and poll if poll_after: time.sleep(poll_after) if proc.poll() is None: return None, None #if still running then return else: logAnyway = True #assume failure and log Logger.warning( "Process is not up after the polling interval " + str(poll_after) + " seconds.") else: return None, None if timeout: q = Queue() t = threading.Timer(timeout, on_timeout, [proc, q]) t.start() #out = proc.communicate()[0].strip('\n') out = proc.communicate() if timeout: if q.empty(): t.cancel() # timeout occurred else: raise ExecuteTimeoutException() code = proc.returncode if (logoutput or logAnyway) and out: Logger.info(out) if throw_on_failure and code: err_msg = Logger.get_protected_text( ("Execution of '%s' returned %d. %s") % (command[-1], code, out)) raise Fail(err_msg) return code, out
def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess32.PIPE, stderr=subprocess32.STDOUT, cwd=None, env=None, preexec_fn=preexec_fn, user=None, wait_for_finish=True, timeout=None, on_timeout=None, path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0, timeout_kill_strategy=TerminateStrategy.TERMINATE_PARENT, returns=[0]): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @param stdout,stderr: subprocess32.PIPE - enable output to variable subprocess32.STDOUT - redirect to stdout None - disable output to variable, and output to Python out straightly (even if logoutput is False) {int fd} - redirect to file with descriptor. {string filename} - redirects to a file with name. """ command_alias = Logger.format_command_for_output(command) command_alias = string_cmd_from_args_list(command_alias) if isinstance( command_alias, (list, tuple)) else command_alias # Append current PATH to env['PATH'] env = _add_current_path_to_env(env) # Append path to env['PATH'] if path: path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path env['PATH'] = os.pathsep.join([env['PATH'], path]) if sudo and user: raise ValueError( "Only one from sudo or user argument could be set to True") # prepare command cmd if sudo: command = as_sudo(command, env=env) elif user: command = as_user(command, user, env=env) # convert to string and escape if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command) # replace placeholder from as_sudo / as_user if present env_str = _get_environment_str(env) for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems(): command = command.replace(placeholder, replacement.format(env_str=env_str)) # --noprofile is used to preserve PATH set for ambari-agent subprocess32_command = [ "/bin/bash", "--login", "--noprofile", "-c", command ] # don't create stdout and stderr pipes, because forked process will not be able to use them if current process dies # creating pipes may lead to the forked process silent crash if not wait_for_finish: stdout = None stderr = None files_to_close = [] if isinstance(stdout, basestring): stdout = open(stdout, 'wb') files_to_close.append(stdout) if isinstance(stderr, basestring): stderr = open(stderr, 'wb') files_to_close.append(stderr) try: proc = subprocess32.Popen(subprocess32_command, stdout=stdout, stderr=stderr, cwd=cwd, env=env, shell=False, close_fds=True, preexec_fn=preexec_fn) if timeout: timeout_event = threading.Event() timer = threading.Timer( timeout, _on_timeout, [proc, timeout_event, timeout_kill_strategy]) timer.start() if not wait_for_finish: return proc # in case logoutput == False, never log. logoutput = logoutput is True and Logger.logger.isEnabledFor( logging.INFO) or logoutput is None and Logger.logger.isEnabledFor( logging.DEBUG) read_set = [] if stdout == subprocess32.PIPE: read_set.append(proc.stdout) if stderr == subprocess32.PIPE: read_set.append(proc.stderr) fd_to_string = {proc.stdout: "", proc.stderr: ""} all_output = "" while read_set: is_proccess_running = proc.poll() is None ready, _, _ = select.select(read_set, [], [], 1) if not is_proccess_running and not ready: break for out_fd in read_set: if out_fd in ready: line = os.read(out_fd.fileno(), 1024) if not line: read_set = copy.copy(read_set) read_set.remove(out_fd) out_fd.close() continue fd_to_string[out_fd] += line all_output += line if on_new_line: try: on_new_line(line, out_fd == proc.stderr) except Exception: err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format( line, traceback.format_exc()) raise Fail(err_msg) if logoutput: sys.stdout.write(line) sys.stdout.flush() # Wait for process to terminate if not timeout or not timeout_event.is_set(): proc.wait() finally: for fp in files_to_close: fp.close() out = fd_to_string[proc.stdout].strip('\n') err = fd_to_string[proc.stderr].strip('\n') all_output = all_output.strip('\n') if timeout: if not timeout_event.is_set(): timer.cancel() # timeout occurred else: err_msg = "Execution of '{0}' was killed due timeout after {1} seconds".format( command, timeout) raise ExecuteTimeoutException(err_msg) code = proc.returncode if throw_on_failure and not code in returns: err_msg = Logger.filter_text( "Execution of '{0}' returned {1}. {2}".format( command_alias, code, all_output)) raise ExecutionFailed(err_msg, code, out, err) # if separate stderr is enabled (by default it's redirected to out) if stderr == subprocess32.PIPE: return code, out, err return code, out