def testSafeSplit(self):
assert SafeSplit('alpha', ' ') == ['alpha']
assert SafeSplit('alpha bravo', ' ') == ['alpha', 'bravo']
assert SafeSplit('alpha bravo charlie',
' ') == ['alpha', 'bravo', 'charlie']
assert SafeSplit('alpha', ' ', 1) == ['alpha', '']
assert SafeSplit('alpha bravo', ' ', 1) == ['alpha', 'bravo']
assert SafeSplit('alpha bravo charlie', ' ',
1) == ['alpha', 'bravo charlie']
assert SafeSplit('alpha', ' ', 1, default=9) == ['alpha', 9]
assert SafeSplit('alpha bravo', ' ', 1,
default=9) == ['alpha', 'bravo']
assert SafeSplit('alpha bravo charlie', ' ', 1,
default=9) == ['alpha', 'bravo charlie']
assert SafeSplit('alpha', ' ', 2) == ['alpha', '', '']
assert SafeSplit('alpha bravo', ' ', 2) == ['alpha', 'bravo', '']
assert SafeSplit('alpha bravo charlie', ' ',
2) == ['alpha', 'bravo', 'charlie']
def Execute(
command_line,
cwd=None,
environ=None,
extra_environ=None,
input=None, # @ReservedAssignment
output_callback=None,
return_code_callback=None,
shell=False,
ignore_auto_quote=False,
clean_eol=True,
pipe_stdout=True,
):
'''
Executes a shell command
:type command_line: list(str) or str
:param command_line:
List of command - line to execute, including the executable as the first element in the
list.
:param str cwd:
The current working directory for the execution.
:type environ: dict(str, str)
:param environ:
The environment variables available for the subprocess. This will replace the current
environment.
If a value is "COPY_FROM_ENVIRON" the value is replaced by the current environment
value before executing the command - line.
This dictionary will be modified by the Execute, so make sure that this is a copy of
your actual data, not the original.
:param dict(str:str) extra_environ:
Environment variables (name, value) to add to the execution environment.
:type input: str | None
:param input:
Text to send as input to the process.
:param callback(str) output_callback:
A optional callback called with the process output as it is generated.
:param callback(int) return_code_callback:
A optional callback called with the execution return -code.
The returned value is ignored.
Because our return value is an iterator, the only way (I found) to give the user access
to the return -code is via callback.
:param bool shell:
From subprocess.py:
If shell is True, the specified command will be executed through the shell.
On UNIX, with shell=False (default): In this case, the Popen class uses os.execvp() to
execute the child program. 'command_line' should normally be a sequence. A string will
be treated as a sequence with the string as the only item (the program to execute).
On UNIX, with shell=True: If 'command_line' is a string, it specifies the command string
to execute through the shell. If 'command_line' is a sequence, the first item specifies
the command string, and any additional items will be treated as additional shell
arguments.
:param bool ignore_auto_quote:
If True, passes the entire command line to subprocess as a single string, instead of
a list of strings.
This is useful when we want to avoid subprocess' algorithm that tries to handle quoting
on its own when receiving a list of arguments.
This way, we are able to use quotes as we wish, without having them escaped by subprocess.
:param bool clean_eol:
If True, output returned and passed to callback will be stripped of eols (\r \n)
:param bool pipe_stdout:
If True, pipe stdout so that it can be returned as a string and passed to the output
callback. If False, stdout will be dumped directly to the console (preserving color),
and the callback will not be called.
:rtype: list(str)
:returns:
Returns the process execution output as a list of strings.
'''
def CmdLineStr(cmd_line):
return ' ' + '\n '.join(cmd_line)
def EnvStr(env):
result = ''
for i, j in sorted(env.items()):
if os.sep in j:
j = '\n * ' + '\n * '.join(sorted(j.split(os.pathsep)))
result += ' - %s = %s\n' % (i, j)
return result
# We accept strings as the command_line.
is_string_command_line = isinstance(command_line, str)
# Handle string/list command_list
if ignore_auto_quote and not is_string_command_line:
# ... with ignore_auto_quote we want a string command_line... but it came as a list
# NOTE: This simple join may cause problems since we can have spaces in a argument. The correct way of
# doing would be something like "shlex.join" (that does not exists, by the way).
command_line = ' '.join(command_line)
elif not ignore_auto_quote and is_string_command_line:
# ... without ignore_auto_quote we want a list command_line... but it came as a string
if sys.platform == 'win32':
command, arguments = SafeSplit(command_line, ' ', 1)
assert command.count(
'"'
) != 1, 'Command may have spaces in it. Use list instead of string.'
# Always use normpath for command, because Windows does not like posix slashes
command = StandardizePath(command)
command = os.path.normpath(command)
command_line = [command] + shlex.split(arguments)
else:
command_line = shlex.split(command_line)
if cwd is None:
cwd = '.'
with Cwd(cwd):
if environ is None:
environ = os.environ.copy()
if extra_environ:
environ.update(extra_environ)
replace_environ = {}
for i_name, i_value in environ.iteritems():
if i_value is COPY_FROM_ENVIRONMENT and i_name in os.environ:
replace_environ[i_name] = os.environ[i_name]
environ.update(replace_environ)
try:
with EnvironmentContextManager(environ):
popen = subprocess.Popen(
command_line,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE if pipe_stdout else None,
stderr=subprocess.STDOUT,
env=environ,
bufsize=0,
shell=shell,
)
except Exception, e:
Reraise(
e, 'While executing "System.Execute":\n'
' environment::\n'
'%s\n'
' current working dir::\n'
' %s\n\n'
' command_line::\n'
'%s\n' %
(EnvStr(environ), os.getcwd(), CmdLineStr(command_line)))
try:
result = []
if popen.stdin:
if input:
try:
popen.stdin.write(input)
except IOError, e:
import errno
if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
raise
popen.stdin.close()
if popen.stdout:
# TODO: EDEN-245: Refactor System.Execute and derivates (git, scons, etc)
if clean_eol: # Read one line at the time, and remove EOLs
for line in iter(popen.stdout.readline, ""):
line = line.rstrip('\n\r')
if output_callback:
output_callback(line)
result.append(line)
else: # Read one char at a time, to keep \r and \n
current_line = ''
carriage = False
for char in iter(lambda: popen.stdout.read(1), ""):
# Check if last line was \r, if not, print what we have
if char != '\n' and carriage:
carriage = False
if output_callback:
output_callback(current_line)
result.append(current_line)
current_line = ''
current_line += char
if char == '\r':
carriage = True
if char == '\n':
if output_callback:
output_callback(current_line)
result.append(current_line)
carriage = False
current_line = ''
def ProcessOpen(
command_line,
cwd=None,
environ=None,
extra_environ=None,
shell=False,
ignore_auto_quote=False,
pipe_stdout=True,
):
'''
Executes a shell command
:type command_line: list(unicode) or unicode
:param command_line:
List of command - line to execute, including the executable as the first element in the
list.
:param unicode cwd:
The current working directory for the execution.
:type environ: dict(unicode, unicode)
:param environ:
The environment variables available for the subprocess. This will replace the current
environment.
If a value is "COPY_FROM_ENVIRON" the value is replaced by the current environment
value before executing the command - line.
This dictionary will be modified by the Execute, so make sure that this is a copy of
your actual data, not the original.
:param dict(unicode:unicode) extra_environ:
Environment variables (name, value) to add to the execution environment.
:param bool shell:
From subprocess.py:
If shell is True, the specified command will be executed through the shell.
On UNIX, with shell=False (default): In this case, the Popen class uses os.execvp() to
execute the child program. 'command_line' should normally be a sequence. A string will
be treated as a sequence with the string as the only item (the program to execute).
On UNIX, with shell=True: If 'command_line' is a string, it specifies the command string
to execute through the shell. If 'command_line' is a sequence, the first item specifies
the command string, and any additional items will be treated as additional shell
arguments.
:param bool ignore_auto_quote:
If True, passes the entire command line to subprocess as a single string, instead of
a list of strings.
This is useful when we want to avoid subprocess' algorithm that tries to handle quoting
on its own when receiving a list of arguments.
This way, we are able to use quotes as we wish, without having them escaped by subprocess.
:param bool pipe_stdout:
If True, pipe stdout so that it can be returned as a string and passed to the output
callback. If False, stdout will be dumped directly to the console (preserving color),
and the callback will not be called.
:returns subprocess.Popen:
'''
locale_encoding = locale.getpreferredencoding()
def CmdLineStr(cmd_line):
if isinstance(cmd_line, unicode):
return ' ' + cmd_line
return ' ' + '\n '.join(cmd_line)
def EncodeWithLocale(value):
if isinstance(value, unicode):
return value.encode(locale_encoding)
if isinstance(value, list):
return [x.encode(locale_encoding) for x in value]
def DecodeWithLocale(value):
if isinstance(value, bytes):
return value.decode(locale_encoding)
if isinstance(value, list):
return [x.decode(locale_encoding) for x in value]
def EnvStr(env):
result = ''
for i, j in sorted(env.items()):
i = i.decode('utf-8')
j = j.decode('utf-8')
if os.sep in j:
j = '\n * ' + '\n * '.join(sorted(j.split(os.pathsep)))
result += ' - %s = %s\n' % (i, j)
return result
# We accept strings as the command_line.
is_string_command_line = isinstance(command_line, unicode)
# Handle string/list command_list
if ignore_auto_quote and not is_string_command_line:
# ... with ignore_auto_quote we want a string command_line... but it came as a list
# NOTE: This simple join may cause problems since we can have spaces in a argument. The correct way of
# doing would be something like "shlex.join" (that does not exists, by the way).
command_line = ' '.join(command_line)
elif not ignore_auto_quote and is_string_command_line:
# ... without ignore_auto_quote we want a list command_line... but it came as a string
if sys.platform == 'win32':
command, arguments = SafeSplit(command_line, ' ', 1)
assert command.count('"') != 1, 'Command may have spaces in it. Use list instead of string.'
# Always use normpath for command, because Windows does not like posix slashes
command = StandardizePath(command)
command = os.path.normpath(command)
# shlex cannot handle non-ascii unicode strings
command_line = [command] + DecodeWithLocale(shlex.split(EncodeWithLocale(arguments)))
else:
# shlex cannot handle non-ascii unicode strings
command_line = DecodeWithLocale(shlex.split(EncodeWithLocale(command_line)))
if cwd is None:
cwd = os.getcwd()
environ = _GetEnviron(environ, extra_environ)
# Make sure the command line is correctly encoded. This always uses locale.getpreferredencoding()
try:
return subprocess.Popen(
EncodeWithLocale(command_line),
cwd=cwd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE if pipe_stdout else None,
stderr=subprocess.STDOUT,
env=environ,
bufsize=0,
shell=shell,
)
except Exception, e:
if isinstance(e, OSError):
# Fix encoding from OSErrors. They also come in locale.getpreferredencoding()
# It's hard to change error messages in OSErrors, so we raise something else.
e = RuntimeError(unicode(str(e), encoding=locale_encoding, errors='replace'))
Reraise(
e,
'While executing "System.Execute":\n'
' environment::\n'
'%s\n'
' current working dir::\n'
' %s\n\n'
' command_line::\n'
'%s\n'
% (EnvStr(environ), cwd, CmdLineStr(command_line))
)
def Execute(
command_line,
cwd=None,
environ=None,
extra_environ=None,
input=None, # @ReservedAssignment
output_callback=None,
return_code_callback=None,
shell=False,
ignore_auto_quote=False,
clean_eol=True,
pipe_stdout=True,
):
'''
Executes a shell command
:type command_line: list(str) or str
:param command_line:
List of command - line to execute, including the executable as the first element in the
list.
:param str cwd:
The current working directory for the execution.
:type environ: dict(str, str)
:param environ:
The environment variables available for the subprocess. This will replace the current
environment.
If a value is "COPY_FROM_ENVIRON" the value is replaced by the current environment
value before executing the command - line.
This dictionary will be modified by the Execute, so make sure that this is a copy of
your actual data, not the original.
:param dict(str:str) extra_environ:
Environment variables (name, value) to add to the execution environment.
:type input: str | None
:param input:
Text to send as input to the process.
:param callback(str) output_callback:
A optional callback called with the process output as it is generated.
:param callback(int) return_code_callback:
A optional callback called with the execution return -code.
The returned value is ignored.
Because our return value is an iterator, the only way (I found) to give the user access
to the return -code is via callback.
:param bool shell:
From subprocess.py:
If shell is True, the specified command will be executed through the shell.
On UNIX, with shell=False (default): In this case, the Popen class uses os.execvp() to
execute the child program. 'command_line' should normally be a sequence. A string will
be treated as a sequence with the string as the only item (the program to execute).
On UNIX, with shell=True: If 'command_line' is a string, it specifies the command string
to execute through the shell. If 'command_line' is a sequence, the first item specifies
the command string, and any additional items will be treated as additional shell
arguments.
:param bool ignore_auto_quote:
If True, passes the entire command line to subprocess as a single string, instead of
a list of strings.
This is useful when we want to avoid subprocess' algorithm that tries to handle quoting
on its own when receiving a list of arguments.
This way, we are able to use quotes as we wish, without having them escaped by subprocess.
:param bool clean_eol:
If True, output returned and passed to callback will be stripped of eols (\r \n)
:param bool pipe_stdout:
If True, pipe stdout so that it can be returned as a string and passed to the output
callback. If False, stdout will be dumped directly to the console (preserving color),
and the callback will not be called.
:rtype: list(str)
:returns:
Returns the process execution output as a list of strings.
'''
def CmdLineStr(cmd_line):
return ' ' + '\n '.join(cmd_line)
def EnvStr(env):
result = ''
for i, j in sorted(env.items()):
if os.sep in j:
j = '\n * ' + '\n * '.join(sorted(j.split(os.pathsep)))
result += ' - %s = %s\n' % (i, j)
return result
# We accept strings as the command_line.
is_string_command_line = isinstance(command_line, str)
# Handle string/list command_list
if ignore_auto_quote and not is_string_command_line:
# ... with ignore_auto_quote we want a string command_line... but it came as a list
# NOTE: This simple join may cause problems since we can have spaces in a argument. The correct way of
# doing would be something like "shlex.join" (that does not exists, by the way).
command_line = ' '.join(command_line)
elif not ignore_auto_quote and is_string_command_line:
# ... without ignore_auto_quote we want a list command_line... but it came as a string
if sys.platform == 'win32':
command, arguments = SafeSplit(command_line, ' ', 1)
assert command.count('"') != 1, 'Command may have spaces in it. Use list instead of string.'
# Always use normpath for command, because Windows does not like posix slashes
command = StandardizePath(command)
command = os.path.normpath(command)
command_line = [command] + shlex.split(arguments)
else:
command_line = shlex.split(command_line)
if cwd is None:
cwd = '.'
with Cwd(cwd):
if environ is None:
environ = os.environ.copy()
if extra_environ:
environ.update(extra_environ)
replace_environ = {}
for i_name, i_value in environ.iteritems():
if i_value is COPY_FROM_ENVIRONMENT and i_name in os.environ:
replace_environ[i_name] = os.environ[i_name]
environ.update(replace_environ)
try:
with EnvironmentContextManager(environ):
popen = subprocess.Popen(
command_line,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE if pipe_stdout else None,
stderr=subprocess.STDOUT,
env=environ,
bufsize=0,
shell=shell,
)
except Exception, e:
Reraise(
e,
'While executing "System.Execute":\n'
' environment::\n'
'%s\n'
' current working dir::\n'
' %s\n\n'
' command_line::\n'
'%s\n'
% (EnvStr(environ), os.getcwd(), CmdLineStr(command_line)))
try:
result = []
if popen.stdin:
if input:
try:
popen.stdin.write(input)
except IOError, e:
import errno
if e.errno != errno.EPIPE and e.errno != errno.EINVAL:
raise
popen.stdin.close()
if popen.stdout:
# TODO: EDEN-245: Refactor System.Execute and derivates (git, scons, etc)
if clean_eol: # Read one line at the time, and remove EOLs
for line in iter(popen.stdout.readline, ""):
line = line.rstrip('\n\r')
if output_callback:
output_callback(line)
result.append(line)
else: # Read one char at a time, to keep \r and \n
current_line = ''
carriage = False
for char in iter(lambda: popen.stdout.read(1), ""):
# Check if last line was \r, if not, print what we have
if char != '\n' and carriage:
carriage = False
if output_callback:
output_callback(current_line)
result.append(current_line)
current_line = ''
current_line += char
if char == '\r':
carriage = True
if char == '\n':
if output_callback:
output_callback(current_line)
result.append(current_line)
carriage = False
current_line = ''
