def _call(command, logoutput=False, throw_on_failure=True,
cwd=None, env=None, preexec_fn=None):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@return: retrun_code, stdout
"""
# convert to string and escape
if isinstance(command, (list, tuple)):
command = ' '.join(pipes.quote(x) for x in command)
command = ["/bin/bash","--login","-c", command]
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=env, shell=False,
preexec_fn=preexec_fn)
out = proc.communicate()[0]
code = proc.returncode
if logoutput and out and out!="":
log.info(out)
if throw_on_failure and code:
err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out)
raise Fail(err_msg)
return code, out
def quote_bash_args(command):
if not command:
return "''"
if not isinstance(command, basestring):
raise Fail("Command should be a list of strings, found '{0}' in command list elements".format(str(command)))
valid = set(string.ascii_letters + string.digits + '@%_-+=:,./')
for char in command:
if char not in valid:
return "'" + command.replace("'", "'\"'\"'") + "'"
return command
def _call(command,
logoutput=False,
throw_on_failure=True,
cwd=None,
env=None,
preexec_fn=None,
user=None,
wait_for_finish=True):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@return: retrun_code, stdout
"""
# convert to string and escape
if isinstance(command, (list, tuple)):
command = ' '.join(pipes.quote(x) for x in command)
"""
Do not su to the supplied user (need to differentiate between when to call su and when not to)
if user:
command = ["su", "-", user, "-c", command]
else:
command = ["/bin/bash","--login","-c", command]
"""
command = ["/bin/bash", "--login", "-c", command]
proc = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=cwd,
env=env,
shell=False,
preexec_fn=preexec_fn)
if not wait_for_finish:
return None, None
out = proc.communicate()[0].strip('\n')
code = proc.returncode
if logoutput and out:
Logger.info(out)
if throw_on_failure and code:
err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code,
out)
raise Fail(err_msg)
return code, out
def _call(command, logoutput=False, throw_on_failure=True,
cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@return: retrun_code, stdout
"""
# convert to string and escape
if isinstance(command, (list, tuple)):
command = ' '.join(quote_bash_args(x) for x in command)
if user:
command = ["su", "-", user, "-c", command]
else:
command = ["/bin/bash","--login","-c", command]
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=env, shell=False,
preexec_fn=preexec_fn)
if not wait_for_finish:
return None, None
if timeout:
q = Queue()
t = threading.Timer( timeout, on_timeout, [proc, q] )
t.start()
out = proc.communicate()[0].strip('\n')
if timeout:
if q.empty():
t.cancel()
# timeout occurred
else:
raise ExecuteTimeoutException()
code = proc.returncode
if logoutput and out:
Logger.info(out)
if throw_on_failure and code:
err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out)
raise Fail(err_msg)
return code, out
def as_sudo(command, env=None, auto_escape=True):
"""
command - list or tuple of arguments.
env - when run as part of Execute resource, this SHOULD NOT be used.
It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if
"""
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command, auto_escape=auto_escape)
else:
# Since ambari user sudoer privileges may be restricted,
# without having /bin/bash permission, and /bin/su permission.
# Running interpreted shell commands in scope of 'sudo' is not possible.
#
# In that case while passing string,
# any bash symbols eventually added to command like && || ; < > | << >> would cause problems.
err_msg = Logger.filter_text(("String command '%s' cannot be run as sudo. Please supply the command as a tuple of arguments") % (command))
raise Fail(err_msg)
env = _get_environment_str(_add_current_path_to_env(env)) if env else ENV_PLACEHOLDER
return "{0} {1} -H -E {2}".format(_get_sudo_binary(), env, command)
def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@param stdout,stderr:
subprocess.PIPE - enable output to variable
subprocess.STDOUT - redirect to stdout
None - disable output to variable, and output to Python out straightly (even if logoutput is False)
{int fd} - redirect to file with descriptor.
{string filename} - redirects to a file with name.
"""
command_alias = Logger.format_command_for_output(command)
command_alias = string_cmd_from_args_list(command_alias) if isinstance(command_alias, (list, tuple)) else command_alias
# Append current PATH to env['PATH']
env = _add_current_path_to_env(env)
# Append path to env['PATH']
if path:
path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path
env['PATH'] = os.pathsep.join([env['PATH'], path])
# prepare command cmd
if sudo:
command = as_sudo(command, env=env)
elif user:
command = as_user(command, user, env=env)
# convert to string and escape
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command)
# replace placeholder from as_sudo / as_user if present
env_str = _get_environment_str(env)
for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
command = command.replace(placeholder, replacement.format(env_str=env_str))
# --noprofile is used to preserve PATH set for ambari-agent
subprocess_command = ["/bin/bash","--login","--noprofile","-c", command]
files_to_close = []
if isinstance(stdout, (basestring)):
stdout = open(stdout, 'wb')
files_to_close.append(stdout)
if isinstance(stderr, (basestring)):
stderr = open(stderr, 'wb')
files_to_close.append(stderr)
try:
proc = subprocess.Popen(subprocess_command, stdout=stdout, stderr=stderr,
cwd=cwd, env=env, shell=False, close_fds=True,
preexec_fn=preexec_fn)
if timeout:
timeout_event = threading.Event()
t = threading.Timer( timeout, _on_timeout, [proc, timeout_event] )
t.start()
if not wait_for_finish:
return proc
# in case logoutput==False, never log.
logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG)
read_set = []
if stdout == subprocess.PIPE:
read_set.append(proc.stdout)
if stderr == subprocess.PIPE:
read_set.append(proc.stderr)
fd_to_string = {
proc.stdout: "",
proc.stderr: ""
}
all_output = ""
while read_set:
is_proccess_running = (proc.poll() == None)
ready, _, _ = select.select(read_set, [], [], 1)
if not is_proccess_running and not ready:
break
for out_fd in read_set:
if out_fd in ready:
line = os.read(out_fd.fileno(), 1024)
if not line:
read_set = copy.copy(read_set)
read_set.remove(out_fd)
out_fd.close()
continue
fd_to_string[out_fd] += line
all_output += line
if on_new_line:
try:
on_new_line(line, out_fd == proc.stderr)
except Exception, err:
err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(line, traceback.format_exc())
raise Fail(err_msg)
if logoutput:
_print(line)
# Wait for process to terminate
if not timeout or not timeout_event.is_set():
proc.wait()
err = fd_to_string[proc.stderr].strip('\n')
all_output = all_output.strip('\n')
if timeout:
if not timeout_event.is_set():
t.cancel()
# timeout occurred
else:
err_msg = Logger.filter_text(("Execution of '%s' was killed due timeout after %d seconds") % (command_alias, timeout))
raise ExecuteTimeoutException(err_msg)
code = proc.returncode
if throw_on_failure and code:
err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias, code, all_output))
raise Fail(err_msg)
# if separate stderr is enabled (by default it's redirected to out)
if stderr == subprocess.PIPE:
return code, out, err
return code, out
def as_sudo(command, env=None, auto_escape=True):
"""
command - list or tuple of arguments.
env - when run as part of Execute resource, this SHOULD NOT be used.
It automatically gets replaced later by call, checked_call. This should be used in not_if, only_if
"""
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command, auto_escape=auto_escape)
def _call(command,
logoutput=None,
throw_on_failure=True,
cwd=None,
env=None,
preexec_fn=None,
user=None,
wait_for_finish=True,
timeout=None,
path=None,
sudo=False,
on_new_line=None):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
"""
command_alias = string_cmd_from_args_list(command) if isinstance(
command, (list, tuple)) else command
# Append current PATH to env['PATH']
env = _add_current_path_to_env(env)
# Append path to env['PATH']
if path:
path = os.pathsep.join(path) if isinstance(path,
(list, tuple)) else path
env['PATH'] = os.pathsep.join([env['PATH'], path])
# prepare command cmd
if sudo:
command = as_sudo(command, env=env)
elif user:
command = as_user(command, user, env=env)
# convert to string and escape
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command)
# replace placeholder from as_sudo / as_user if present
env_str = _get_environment_str(env)
for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
command = command.replace(placeholder,
replacement.format(env_str=env_str))
master_fd, slave_fd = pty.openpty()
# --noprofile is used to preserve PATH set for ambari-agent
subprocess_command = ["/bin/bash", "--login", "--noprofile", "-c", command]
proc = subprocess.Popen(subprocess_command,
bufsize=1,
stdout=slave_fd,
stderr=subprocess.STDOUT,
cwd=cwd,
env=env,
shell=False,
preexec_fn=preexec_fn)
if timeout:
timeout_event = threading.Event()
t = threading.Timer(timeout, _on_timeout, [proc, timeout_event])
t.start()
if not wait_for_finish:
return proc
# in case logoutput==False, never log.
logoutput = logoutput == True and Logger.logger.isEnabledFor(
logging.INFO) or logoutput == None and Logger.logger.isEnabledFor(
logging.DEBUG)
out = ""
read_timeout = .04 # seconds
try:
while True:
ready, _, _ = select.select([master_fd], [], [], read_timeout)
if ready:
line = os.read(master_fd, 512)
if not line:
break
out += line
try:
if on_new_line:
on_new_line(line)
except Exception, err:
err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(
line, traceback.format_exc())
raise Fail(err_msg)
if logoutput:
_print(line)
elif proc.poll() is not None:
break # proc exited
def _call(command,
logoutput=False,
throw_on_failure=True,
cwd=None,
env=None,
preexec_fn=None,
user=None,
wait_for_finish=True,
timeout=None,
pid_file_name=None,
poll_after=None):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@return: retrun_code, stdout
"""
# convert to string and escape
if isinstance(command, (list, tuple)):
command = ' '.join(quote_bash_args(x) for x in command)
"""
Do not su to the supplied user (need to differentiate between when to call su and when not to)
if user:
command = ["su", "-", user, "-c", command]
else:
command = ["/bin/bash","--login","-c", command]
"""
command = ["/bin/bash", "--login", "-c", command]
#adding redirecting stdout stderr to file
outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
stdoutFile = open(outfilename, 'w+')
stderrFile = open(errfilename, 'w+')
proc = subprocess.Popen(command,
stdout=stdoutFile,
stderr=stderrFile,
universal_newlines=True,
cwd=cwd,
env=env,
shell=False,
preexec_fn=preexec_fn)
logAnyway = False
if not wait_for_finish:
if pid_file_name:
pidfile = open(pid_file_name, 'w')
pidfile.write(str(proc.pid))
pidfile.close()
## wait poll_after seconds and poll
if poll_after:
time.sleep(poll_after)
if proc.poll() is None:
return None, None #if still running then return
else:
logAnyway = True #assume failure and log
Logger.warning(
"Process is not up after the polling interval " +
str(poll_after) + " seconds.")
else:
return None, None
if timeout:
q = Queue()
t = threading.Timer(timeout, on_timeout, [proc, q])
t.start()
#out = proc.communicate()[0].strip('\n')
out = proc.communicate()
if timeout:
if q.empty():
t.cancel()
# timeout occurred
else:
raise ExecuteTimeoutException()
code = proc.returncode
if (logoutput or logAnyway) and out:
Logger.info(out)
if throw_on_failure and code:
err_msg = Logger.get_protected_text(
("Execution of '%s' returned %d. %s") % (command[-1], code, out))
raise Fail(err_msg)
return code, out
def _call(command,
logoutput=None,
throw_on_failure=True,
stdout=subprocess32.PIPE,
stderr=subprocess32.STDOUT,
cwd=None,
env=None,
preexec_fn=preexec_fn,
user=None,
wait_for_finish=True,
timeout=None,
on_timeout=None,
path=None,
sudo=False,
on_new_line=None,
tries=1,
try_sleep=0,
timeout_kill_strategy=TerminateStrategy.TERMINATE_PARENT,
returns=[0]):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@param stdout,stderr:
subprocess32.PIPE - enable output to variable
subprocess32.STDOUT - redirect to stdout
None - disable output to variable, and output to Python out straightly (even if logoutput is False)
{int fd} - redirect to file with descriptor.
{string filename} - redirects to a file with name.
"""
command_alias = Logger.format_command_for_output(command)
command_alias = string_cmd_from_args_list(command_alias) if isinstance(
command_alias, (list, tuple)) else command_alias
# Append current PATH to env['PATH']
env = _add_current_path_to_env(env)
# Append path to env['PATH']
if path:
path = os.pathsep.join(path) if isinstance(path,
(list, tuple)) else path
env['PATH'] = os.pathsep.join([env['PATH'], path])
if sudo and user:
raise ValueError(
"Only one from sudo or user argument could be set to True")
# prepare command cmd
if sudo:
command = as_sudo(command, env=env)
elif user:
command = as_user(command, user, env=env)
# convert to string and escape
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command)
# replace placeholder from as_sudo / as_user if present
env_str = _get_environment_str(env)
for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
command = command.replace(placeholder,
replacement.format(env_str=env_str))
# --noprofile is used to preserve PATH set for ambari-agent
subprocess32_command = [
"/bin/bash", "--login", "--noprofile", "-c", command
]
# don't create stdout and stderr pipes, because forked process will not be able to use them if current process dies
# creating pipes may lead to the forked process silent crash
if not wait_for_finish:
stdout = None
stderr = None
files_to_close = []
if isinstance(stdout, basestring):
stdout = open(stdout, 'wb')
files_to_close.append(stdout)
if isinstance(stderr, basestring):
stderr = open(stderr, 'wb')
files_to_close.append(stderr)
try:
proc = subprocess32.Popen(subprocess32_command,
stdout=stdout,
stderr=stderr,
cwd=cwd,
env=env,
shell=False,
close_fds=True,
preexec_fn=preexec_fn)
if timeout:
timeout_event = threading.Event()
timer = threading.Timer(
timeout, _on_timeout,
[proc, timeout_event, timeout_kill_strategy])
timer.start()
if not wait_for_finish:
return proc
# in case logoutput == False, never log.
logoutput = logoutput is True and Logger.logger.isEnabledFor(
logging.INFO) or logoutput is None and Logger.logger.isEnabledFor(
logging.DEBUG)
read_set = []
if stdout == subprocess32.PIPE:
read_set.append(proc.stdout)
if stderr == subprocess32.PIPE:
read_set.append(proc.stderr)
fd_to_string = {proc.stdout: "", proc.stderr: ""}
all_output = ""
while read_set:
is_proccess_running = proc.poll() is None
ready, _, _ = select.select(read_set, [], [], 1)
if not is_proccess_running and not ready:
break
for out_fd in read_set:
if out_fd in ready:
line = os.read(out_fd.fileno(), 1024)
if not line:
read_set = copy.copy(read_set)
read_set.remove(out_fd)
out_fd.close()
continue
fd_to_string[out_fd] += line
all_output += line
if on_new_line:
try:
on_new_line(line, out_fd == proc.stderr)
except Exception:
err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(
line, traceback.format_exc())
raise Fail(err_msg)
if logoutput:
sys.stdout.write(line)
sys.stdout.flush()
# Wait for process to terminate
if not timeout or not timeout_event.is_set():
proc.wait()
finally:
for fp in files_to_close:
fp.close()
out = fd_to_string[proc.stdout].strip('\n')
err = fd_to_string[proc.stderr].strip('\n')
all_output = all_output.strip('\n')
if timeout:
if not timeout_event.is_set():
timer.cancel()
# timeout occurred
else:
err_msg = "Execution of '{0}' was killed due timeout after {1} seconds".format(
command, timeout)
raise ExecuteTimeoutException(err_msg)
code = proc.returncode
if throw_on_failure and not code in returns:
err_msg = Logger.filter_text(
"Execution of '{0}' returned {1}. {2}".format(
command_alias, code, all_output))
raise ExecutionFailed(err_msg, code, out, err)
# if separate stderr is enabled (by default it's redirected to out)
if stderr == subprocess32.PIPE:
return code, out, err
return code, out