def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@param stdout,stderr:
subprocess.PIPE - enable output to variable
subprocess.STDOUT - redirect to stdout
None - disable output to variable, and output to Python out straightly (even if logoutput is False)
{int fd} - redirect to file with descriptor.
{string filename} - redirects to a file with name.
"""
command_alias = Logger.format_command_for_output(command)
command_alias = string_cmd_from_args_list(command_alias) if isinstance(command_alias, (list, tuple)) else command_alias
# Append current PATH to env['PATH']
env = _add_current_path_to_env(env)
# Append path to env['PATH']
if path:
path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path
env['PATH'] = os.pathsep.join([env['PATH'], path])
# prepare command cmd
if sudo:
command = as_sudo(command, env=env)
elif user:
command = as_user(command, user, env=env)
# convert to string and escape
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command)
# replace placeholder from as_sudo / as_user if present
env_str = _get_environment_str(env)
for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
command = command.replace(placeholder, replacement.format(env_str=env_str))
# --noprofile is used to preserve PATH set for ambari-agent
subprocess_command = ["/bin/bash","--login","--noprofile","-c", command]
files_to_close = []
if isinstance(stdout, (basestring)):
stdout = open(stdout, 'wb')
files_to_close.append(stdout)
if isinstance(stderr, (basestring)):
stderr = open(stderr, 'wb')
files_to_close.append(stderr)
try:
proc = subprocess.Popen(subprocess_command, stdout=stdout, stderr=stderr,
cwd=cwd, env=env, shell=False, close_fds=True,
preexec_fn=preexec_fn)
if timeout:
timeout_event = threading.Event()
t = threading.Timer( timeout, _on_timeout, [proc, timeout_event] )
t.start()
if not wait_for_finish:
return proc
# in case logoutput==False, never log.
logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG)
read_set = []
if stdout == subprocess.PIPE:
read_set.append(proc.stdout)
if stderr == subprocess.PIPE:
read_set.append(proc.stderr)
fd_to_string = {
proc.stdout: "",
proc.stderr: ""
}
all_output = ""
while read_set:
is_proccess_running = (proc.poll() == None)
ready, _, _ = select.select(read_set, [], [], 1)
if not is_proccess_running and not ready:
break
for out_fd in read_set:
if out_fd in ready:
line = os.read(out_fd.fileno(), 1024)
if not line:
read_set = copy.copy(read_set)
read_set.remove(out_fd)
out_fd.close()
continue
fd_to_string[out_fd] += line
all_output += line
if on_new_line:
try:
on_new_line(line, out_fd == proc.stderr)
except Exception, err:
err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(line, traceback.format_exc())
raise Fail(err_msg)
if logoutput:
_print(line)
# Wait for process to terminate
if not timeout or not timeout_event.is_set():
proc.wait()
def _call(command, logoutput=None, throw_on_failure=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT,
cwd=None, env=None, preexec_fn=None, user=None, wait_for_finish=True, timeout=None, on_timeout=None,
path=None, sudo=False, on_new_line=None, tries=1, try_sleep=0):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@param stdout,stderr:
subprocess.PIPE - enable output to variable
subprocess.STDOUT - redirect to stdout
None - disable output to variable, and output to Python out straightly (even if logoutput is False)
{int fd} - redirect to file with descriptor.
{string filename} - redirects to a file with name.
"""
command_alias = Logger.format_command_for_output(command)
command_alias = string_cmd_from_args_list(command_alias) if isinstance(command_alias, (list, tuple)) else command_alias
# Append current PATH to env['PATH']
env = _add_current_path_to_env(env)
# Append path to env['PATH']
if path:
path = os.pathsep.join(path) if isinstance(path, (list, tuple)) else path
env['PATH'] = os.pathsep.join([env['PATH'], path])
# prepare command cmd
if sudo:
command = as_sudo(command, env=env)
elif user:
command = as_user(command, user, env=env)
# convert to string and escape
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command)
# replace placeholder from as_sudo / as_user if present
env_str = _get_environment_str(env)
for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
command = command.replace(placeholder, replacement.format(env_str=env_str))
# --noprofile is used to preserve PATH set for ambari-agent
subprocess_command = ["/bin/bash","--login","--noprofile","-c", command]
files_to_close = []
if isinstance(stdout, (basestring)):
stdout = open(stdout, 'wb')
files_to_close.append(stdout)
if isinstance(stderr, (basestring)):
stderr = open(stderr, 'wb')
files_to_close.append(stderr)
try:
proc = subprocess.Popen(subprocess_command, stdout=stdout, stderr=stderr,
cwd=cwd, env=env, shell=False, close_fds=True,
preexec_fn=preexec_fn)
if timeout:
timeout_event = threading.Event()
t = threading.Timer( timeout, _on_timeout, [proc, timeout_event] )
t.start()
if not wait_for_finish:
return proc
# in case logoutput==False, never log.
logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None and Logger.logger.isEnabledFor(logging.DEBUG)
read_set = []
if stdout == subprocess.PIPE:
read_set.append(proc.stdout)
if stderr == subprocess.PIPE:
read_set.append(proc.stderr)
fd_to_string = {
proc.stdout: "",
proc.stderr: ""
}
all_output = ""
while read_set:
is_proccess_running = (proc.poll() == None)
ready, _, _ = select.select(read_set, [], [], 1)
if not is_proccess_running and not ready:
break
for out_fd in read_set:
if out_fd in ready:
line = os.read(out_fd.fileno(), 1024)
if not line:
read_set = copy.copy(read_set)
read_set.remove(out_fd)
out_fd.close()
continue
fd_to_string[out_fd] += line
all_output += line
if on_new_line:
try:
on_new_line(line, out_fd == proc.stderr)
except Exception, err:
err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(line, traceback.format_exc())
raise Fail(err_msg)
if logoutput:
_print(line)
# Wait for process to terminate
if not timeout or not timeout_event.is_set():
proc.wait()
def _call(command,
logoutput=None,
throw_on_failure=True,
stdout=subprocess32.PIPE,
stderr=subprocess32.STDOUT,
cwd=None,
env=None,
preexec_fn=preexec_fn,
user=None,
wait_for_finish=True,
timeout=None,
on_timeout=None,
path=None,
sudo=False,
on_new_line=None,
tries=1,
try_sleep=0,
timeout_kill_strategy=TerminateStrategy.TERMINATE_PARENT,
returns=[0]):
"""
Execute shell command
@param command: list/tuple of arguments (recommended as more safe - don't need to escape)
or string of the command to execute
@param logoutput: boolean, whether command output should be logged of not
@param throw_on_failure: if true, when return code is not zero exception is thrown
@param stdout,stderr:
subprocess32.PIPE - enable output to variable
subprocess32.STDOUT - redirect to stdout
None - disable output to variable, and output to Python out straightly (even if logoutput is False)
{int fd} - redirect to file with descriptor.
{string filename} - redirects to a file with name.
"""
command_alias = Logger.format_command_for_output(command)
command_alias = string_cmd_from_args_list(command_alias) if isinstance(
command_alias, (list, tuple)) else command_alias
# Append current PATH to env['PATH']
env = _add_current_path_to_env(env)
# Append path to env['PATH']
if path:
path = os.pathsep.join(path) if isinstance(path,
(list, tuple)) else path
env['PATH'] = os.pathsep.join([env['PATH'], path])
if sudo and user:
raise ValueError(
"Only one from sudo or user argument could be set to True")
# prepare command cmd
if sudo:
command = as_sudo(command, env=env)
elif user:
command = as_user(command, user, env=env)
# convert to string and escape
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command)
# replace placeholder from as_sudo / as_user if present
env_str = _get_environment_str(env)
for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
command = command.replace(placeholder,
replacement.format(env_str=env_str))
# --noprofile is used to preserve PATH set for ambari-agent
subprocess32_command = [
"/bin/bash", "--login", "--noprofile", "-c", command
]
# don't create stdout and stderr pipes, because forked process will not be able to use them if current process dies
# creating pipes may lead to the forked process silent crash
if not wait_for_finish:
stdout = None
stderr = None
files_to_close = []
if isinstance(stdout, basestring):
stdout = open(stdout, 'wb')
files_to_close.append(stdout)
if isinstance(stderr, basestring):
stderr = open(stderr, 'wb')
files_to_close.append(stderr)
try:
proc = subprocess32.Popen(subprocess32_command,
stdout=stdout,
stderr=stderr,
cwd=cwd,
env=env,
shell=False,
close_fds=True,
preexec_fn=preexec_fn)
if timeout:
timeout_event = threading.Event()
timer = threading.Timer(
timeout, _on_timeout,
[proc, timeout_event, timeout_kill_strategy])
timer.start()
if not wait_for_finish:
return proc
# in case logoutput == False, never log.
logoutput = logoutput is True and Logger.logger.isEnabledFor(
logging.INFO) or logoutput is None and Logger.logger.isEnabledFor(
logging.DEBUG)
read_set = []
if stdout == subprocess32.PIPE:
read_set.append(proc.stdout)
if stderr == subprocess32.PIPE:
read_set.append(proc.stderr)
fd_to_string = {proc.stdout: "", proc.stderr: ""}
all_output = ""
while read_set:
is_proccess_running = proc.poll() is None
ready, _, _ = select.select(read_set, [], [], 1)
if not is_proccess_running and not ready:
break
for out_fd in read_set:
if out_fd in ready:
line = os.read(out_fd.fileno(), 1024)
if not line:
read_set = copy.copy(read_set)
read_set.remove(out_fd)
out_fd.close()
continue
fd_to_string[out_fd] += line
all_output += line
if on_new_line:
try:
on_new_line(line, out_fd == proc.stderr)
except Exception:
err_msg = "Caused by on_new_line function failed with exception for input argument '{0}':\n{1}".format(
line, traceback.format_exc())
raise Fail(err_msg)
if logoutput:
sys.stdout.write(line)
sys.stdout.flush()
# Wait for process to terminate
if not timeout or not timeout_event.is_set():
proc.wait()
finally:
for fp in files_to_close:
fp.close()
out = fd_to_string[proc.stdout].strip('\n')
err = fd_to_string[proc.stderr].strip('\n')
all_output = all_output.strip('\n')
if timeout:
if not timeout_event.is_set():
timer.cancel()
# timeout occurred
else:
err_msg = "Execution of '{0}' was killed due timeout after {1} seconds".format(
command, timeout)
raise ExecuteTimeoutException(err_msg)
code = proc.returncode
if throw_on_failure and not code in returns:
err_msg = Logger.filter_text(
"Execution of '{0}' returned {1}. {2}".format(
command_alias, code, all_output))
raise ExecutionFailed(err_msg, code, out, err)
# if separate stderr is enabled (by default it's redirected to out)
if stderr == subprocess32.PIPE:
return code, out, err
return code, out
