def _call(command, logoutput=False, throw_on_failure=True, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @return: retrun_code, stdout """ # convert to string and escape if isinstance(command, (list, tuple)): command = ' '.join(quote_bash_args(x) for x in command) if user: command = ["su", "-", user, "-c", command] else: command = ["/bin/bash","--login","-c", command] proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) if not wait_for_finish: return None, None if timeout: q = Queue() t = threading.Timer( timeout, on_timeout, [proc, q] ) t.start() out = proc.communicate()[0].strip('\n') if timeout: if q.empty(): t.cancel() # timeout occurred else: raise ExecuteTimeoutException() code = proc.returncode if logoutput and out: Logger.info(out) if throw_on_failure and code: err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out) raise Fail(err_msg) return code, out
finally: for fp in files_to_close: fp.close() out = fd_to_string[proc.stdout].strip('\n') err = fd_to_string[proc.stderr].strip('\n') all_output = all_output.strip('\n') if timeout: if not timeout_event.is_set(): t.cancel() # timeout occurred else: err_msg = Logger.filter_text(("Execution of '%s' was killed due timeout after %d seconds") % (command_alias, timeout)) raise ExecuteTimeoutException(err_msg) code = proc.returncode if throw_on_failure and code: err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias, code, all_output)) raise Fail(err_msg) # if separate stderr is enabled (by default it's redirected to out) if stderr == subprocess.PIPE: return code, out, err return code, out def as_sudo(command, env=None, auto_escape=True): """
_print(line) elif proc.poll() is not None: break # proc exited finally: os.close(slave_fd) os.close(master_fd) proc.wait() out = out.strip('\n') if timeout: if not timeout_event.is_set(): t.cancel() # timeout occurred else: raise ExecuteTimeoutException() code = proc.returncode if throw_on_failure and code: err_msg = Logger.filter_text( ("Execution of '%s' returned %d. %s") % (command_alias, code, out)) raise Fail(err_msg) return code, out def as_sudo(command, env=None): """ command - list or tuple of arguments. env - when run as part of Execute resource, this SHOULD NOT be used.
def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess32.PIPE, stderr=subprocess32.STDOUT, cwd=None, env=None, preexec_fn=preexec_fn, user=None, wait_for_finish=True, timeout=None, on_timeout=None, path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0, timeout_kill_strategy=TerminateStrategy.TERMINATE_PARENT, returns=[0]): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @param stdout,stderr: subprocess32.PIPE - enable output to variable subprocess32.STDOUT - redirect to stdout None - disable output to variable, and output to Python out straightly (even if logoutput is False) {int fd} - redirect to file with descriptor. {string filename} - redirects to a file with name. """ command_alias = Logger.format_command_for_output(command) command_alias = string_cmd_from_args_list(command_alias) if isinstance( command_alias, (list, tuple)) else command_alias # Append current PATH to env['PATH'] env = _add_current_path_to_env(env) # Append path to env['PATH'] if path: path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path env['PATH'] = os.pathsep.join([env['PATH'], path]) if sudo and user: raise ValueError( "Only one from sudo or user argument could be set to True") # prepare command cmd if sudo: command = as_sudo(command, env=env) elif user: command = as_user(command, user, env=env) # convert to string and escape if isinstance(command, (list, tuple)): command = string_cmd_from_args_list(command) # replace placeholder from as_sudo / as_user if present env_str = _get_environment_str(env) for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems(): command = command.replace(placeholder, replacement.format(env_str=env_str)) # --noprofile is used to preserve PATH set for ambari-agent subprocess32_command = [ "/bin/bash", "--login", "--noprofile", "-c", command ] files_to_close = [] if isinstance(stdout, basestring): stdout = open(stdout, 'wb') files_to_close.append(stdout) if isinstance(stderr, basestring): stderr = open(stderr, 'wb') files_to_close.append(stderr) try: proc = subprocess32.Popen(subprocess32_command, stdout=stdout, stderr=stderr, cwd=cwd, env=env, shell=False, close_fds=True, preexec_fn=preexec_fn) if timeout: timeout_event = threading.Event() timer = threading.Timer( timeout, _on_timeout, [proc, timeout_event, timeout_kill_strategy]) timer.start() if not wait_for_finish: return proc # in case logoutput == False, never log. logoutput = logoutput is True and Logger.logger.isEnabledFor( logging.INFO) or logoutput is None and Logger.logger.isEnabledFor( logging.DEBUG) read_set = [] if stdout == subprocess32.PIPE: read_set.append(proc.stdout) if stderr == subprocess32.PIPE: read_set.append(proc.stderr) fd_to_string = {proc.stdout: "", proc.stderr: ""} all_output = "" while read_set: is_proccess_running = proc.poll() is None ready, _, _ = select.select(read_set, [], [], 1) if not is_proccess_running and not ready: break for out_fd in read_set: if out_fd in ready: line = os.read(out_fd.fileno(), 1024) if not line: read_set = copy.copy(read_set) read_set.remove(out_fd) out_fd.close() continue fd_to_string[out_fd] += line all_output += line if on_new_line: try: on_new_line(line, out_fd == proc.stderr) except Exception: err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format( line, traceback.format_exc()) raise Fail(err_msg) if logoutput: sys.stdout.write(line) sys.stdout.flush() # Wait for process to terminate if not timeout or not timeout_event.is_set(): proc.wait() finally: for fp in files_to_close: fp.close() out = fd_to_string[proc.stdout].strip('\n') err = fd_to_string[proc.stderr].strip('\n') all_output = all_output.strip('\n') if timeout: if not timeout_event.is_set(): timer.cancel() # timeout occurred else: err_msg = "Execution of '{0}' was killed due timeout after {1} seconds".format( command, timeout) raise ExecuteTimeoutException(err_msg) code = proc.returncode if throw_on_failure and not code in returns: err_msg = Logger.filter_text( "Execution of '{0}' returned {1}. {2}".format( command_alias, code, all_output)) raise ExecutionFailed(err_msg, code, out, err) # if separate stderr is enabled (by default it's redirected to out) if stderr == subprocess32.PIPE: return code, out, err return code, out
def _call(command, logoutput=False, throw_on_failure=True, cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, pid_file_name=None, poll_after=None): """ Execute shell command @param command: list/tuple of arguments (recommended as more safe - don't need to escape) or string of the command to execute @param logoutput: boolean, whether command output should be logged of not @param throw_on_failure: if true, when return code is not zero exception is thrown @return: retrun_code, stdout """ # convert to string and escape if isinstance(command, (list, tuple)): command = ' '.join(quote_bash_args(x) for x in command) """ Do not su to the supplied user (need to differentiate between when to call su and when not to) if user: command = ["su", "-", user, "-c", command] else: command = ["/bin/bash","--login","-c", command] """ command = ["/bin/bash", "--login", "-c", command] #adding redirecting stdout stderr to file outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE stdoutFile = open(outfilename, 'w+') stderrFile = open(errfilename, 'w+') proc = subprocess.Popen(command, stdout=stdoutFile, stderr=stderrFile, universal_newlines=True, cwd=cwd, env=env, shell=False, preexec_fn=preexec_fn) logAnyway = False if not wait_for_finish: if pid_file_name: pidfile = open(pid_file_name, 'w') pidfile.write(str(proc.pid)) pidfile.close() ## wait poll_after seconds and poll if poll_after: time.sleep(poll_after) if proc.poll() is None: return None, None #if still running then return else: logAnyway = True #assume failure and log Logger.warning( "Process is not up after the polling interval " + str(poll_after) + " seconds.") else: return None, None if timeout: q = Queue() t = threading.Timer(timeout, on_timeout, [proc, q]) t.start() #out = proc.communicate()[0].strip('\n') out = proc.communicate() if timeout: if q.empty(): t.cancel() # timeout occurred else: raise ExecuteTimeoutException() code = proc.returncode if (logoutput or logAnyway) and out: Logger.info(out) if throw_on_failure and code: err_msg = Logger.get_protected_text( ("Execution of '%s' returned %d. %s") % (command[-1], code, out)) raise Fail(err_msg) return code, out