def _run_command(self, args=None, cwd=None, append_env=None,
explicit_env=None, log_name=None, log_level=logging.INFO,
line_handler=None, require_unix_environment=False,
ignore_errors=False, ignore_children=False):
"""Runs a single command to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'process', {'args': args}, ' '.join(args))
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.strip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
processOutputLine=[handleLine], universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = p.wait()
if status != 0 and not ignore_errors:
raise Exception('Process executed with non-0 exit code: %s' % args)
def InvokeClWithDependencyGeneration(cmdline):
target = ""
# Figure out what the target is
for arg in cmdline:
if arg.startswith("-Fo"):
target = arg[3:]
break
if target is None:
print >> sys.stderr, "No target set"
return 1
# Assume the source file is the last argument
source = cmdline[-1]
assert not source.startswith('-')
# The deps target lives here
depstarget = os.path.basename(target) + ".pp"
showincludes = '-showIncludes' in cmdline
cmdline += ['-showIncludes']
mk = Makefile()
rule = mk.create_rule([target])
rule.add_dependencies([normcase(source)])
def on_line(line):
# cl -showIncludes prefixes every header with "Note: including file:"
# and an indentation corresponding to the depth (which we don't need)
if line.startswith(CL_INCLUDES_PREFIX):
dep = line[len(CL_INCLUDES_PREFIX):].strip()
# We can't handle pathes with spaces properly in mddepend.pl, but
# we can assume that anything in a path with spaces is a system
# header and throw it away.
dep = normcase(dep)
if ' ' not in dep:
rule.add_dependencies([dep])
# Hide the line by returning early
if not showincludes:
return
# Make sure we preserve the relevant output from cl. mozprocess
# swallows the newline delimiter, so we need to re-add it.
sys.stdout.write(line)
sys.stdout.write('\n')
# We need to ignore children because MSVC can fire up a background process
# during compilation. This process is cleaned up on its own. If we kill it,
# we can run into weird compilation issues.
p = ProcessHandlerMixin(cmdline,
processOutputLine=[on_line],
ignore_children=True)
p.run()
p.processOutput()
ret = p.wait()
if ret != 0 or target == "":
# p.wait() returns a long. Somehow sys.exit(long(0)) is like
# sys.exit(1). Don't ask why.
return int(ret)
depsdir = os.path.normpath(os.path.join(os.curdir, ".deps"))
depstarget = os.path.join(depsdir, depstarget)
if not os.path.isdir(depsdir):
try:
os.makedirs(depsdir)
except OSError:
pass # This suppresses the error we get when the dir exists, at the
# cost of masking failure to create the directory. We'll just
# die on the next line though, so it's not that much of a loss.
with open(depstarget, "w") as f:
mk.dump(f)
return 0
def run_process(self,
args=None,
cwd=None,
append_env=None,
explicit_env=None,
log_name=None,
log_level=logging.INFO,
line_handler=None,
require_unix_environment=False,
ensure_exit_code=0,
ignore_children=False,
pass_thru=False):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
ensure_exit_code is used to ensure the exit code of a process matches
what is expected. If it is an integer, we raise an Exception if the
exit code does not match this value. If it is True, we ensure the exit
code is 0. If it is False, we don't perform any exit code validation.
pass_thru is a special execution mode where the child process inherits
this process's standard file handles (stdin, stdout, stderr) as well as
additional file descriptors. It should be used for interactive processes
where buffering from mozprocess could be an issue. pass_thru does not
use mozprocess. Therefore, arguments like log_name, line_handler,
and ignore_children have no effect.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args))
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.strip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
self.log(logging.DEBUG, 'process', {'env': use_env},
'Environment: {env}')
if pass_thru:
status = subprocess.call(args, cwd=cwd, env=use_env)
else:
p = ProcessHandlerMixin(args,
cwd=cwd,
env=use_env,
processOutputLine=[handleLine],
universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = p.wait()
if ensure_exit_code is False:
return status
if ensure_exit_code is True:
ensure_exit_code = 0
if status != ensure_exit_code:
raise Exception('Process executed with non-0 exit code: %s' % args)
return status
def run_process(self,
args=None,
cwd=None,
append_env=None,
explicit_env=None,
log_name=None,
log_level=logging.INFO,
line_handler=None,
require_unix_environment=False,
ensure_exit_code=0,
ignore_children=False,
pass_thru=False,
python_unbuffered=True):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
ensure_exit_code is used to ensure the exit code of a process matches
what is expected. If it is an integer, we raise an Exception if the
exit code does not match this value. If it is True, we ensure the exit
code is 0. If it is False, we don't perform any exit code validation.
pass_thru is a special execution mode where the child process inherits
this process's standard file handles (stdin, stdout, stderr) as well as
additional file descriptors. It should be used for interactive processes
where buffering from mozprocess could be an issue. pass_thru does not
use mozprocess. Therefore, arguments like log_name, line_handler,
and ignore_children have no effect.
When python_unbuffered is set, the PYTHONUNBUFFERED environment variable
will be set in the child process. This is normally advantageous (see bug
1627873) but is detrimental in certain circumstances (specifically, we
have seen issues when using pass_thru mode to open a Python subshell, as
in bug 1628838). This variable should be set to False to avoid bustage
in those circumstances.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'new_process', {'args': ' '.join(args)},
'{args}')
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.rstrip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
if python_unbuffered:
use_env['PYTHONUNBUFFERED'] = '1'
self.log(logging.DEBUG, 'process', {'env': use_env},
'Environment: {env}')
use_env = ensure_subprocess_env(use_env)
if pass_thru:
proc = subprocess.Popen(args, cwd=cwd, env=use_env)
status = None
# Leave it to the subprocess to handle Ctrl+C. If it terminates as
# a result of Ctrl+C, proc.wait() will return a status code, and,
# we get out of the loop. If it doesn't, like e.g. gdb, we continue
# waiting.
while status is None:
try:
status = proc.wait()
except KeyboardInterrupt:
pass
else:
p = ProcessHandlerMixin(args,
cwd=cwd,
env=use_env,
processOutputLine=[handleLine],
universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = None
sig = None
while status is None:
try:
if sig is None:
status = p.wait()
else:
status = p.kill(sig=sig)
except KeyboardInterrupt:
if sig is None:
sig = signal.SIGINT
elif sig == signal.SIGINT:
# If we've already tried SIGINT, escalate.
sig = signal.SIGKILL
if ensure_exit_code is False:
return status
if ensure_exit_code is True:
ensure_exit_code = 0
if status != ensure_exit_code:
raise Exception('Process executed with non-0 exit code %d: %s' %
(status, args))
return status
def run_process(self,
args=None,
cwd=None,
append_env=None,
explicit_env=None,
log_name=None,
log_level=logging.INFO,
line_handler=None,
require_unix_environment=False,
ensure_exit_code=0,
ignore_children=False,
pass_thru=False):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
ensure_exit_code is used to ensure the exit code of a process matches
what is expected. If it is an integer, we raise an Exception if the
exit code does not match this value. If it is True, we ensure the exit
code is 0. If it is False, we don't perform any exit code validation.
pass_thru is a special execution mode where the child process inherits
this process's standard file handles (stdin, stdout, stderr) as well as
additional file descriptors. It should be used for interactive processes
where buffering from mozprocess could be an issue. pass_thru does not
use mozprocess. Therefore, arguments like log_name, line_handler,
and ignore_children have no effect.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'new_process', {'args': ' '.join(args)},
'{args}')
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.rstrip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
self.log(logging.DEBUG, 'process', {'env': use_env},
'Environment: {env}')
# There is a bug in subprocess where it doesn't like unicode types in
# environment variables. Here, ensure all unicode are converted to
# binary. utf-8 is our globally assumed default. If the caller doesn't
# want UTF-8, they shouldn't pass in a unicode instance.
normalized_env = {}
for k, v in use_env.items():
if isinstance(k, unicode):
k = k.encode('utf-8', 'strict')
if isinstance(v, unicode):
v = v.encode('utf-8', 'strict')
normalized_env[k] = v
use_env = normalized_env
if pass_thru:
proc = subprocess.Popen(args, cwd=cwd, env=use_env)
status = None
# Leave it to the subprocess to handle Ctrl+C. If it terminates as
# a result of Ctrl+C, proc.wait() will return a status code, and,
# we get out of the loop. If it doesn't, like e.g. gdb, we continue
# waiting.
while status is None:
try:
status = proc.wait()
except KeyboardInterrupt:
pass
else:
p = ProcessHandlerMixin(args,
cwd=cwd,
env=use_env,
processOutputLine=[handleLine],
universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = None
while status is None:
try:
status = p.wait()
except KeyboardInterrupt:
status = p.kill()
if ensure_exit_code is False:
return status
if ensure_exit_code is True:
ensure_exit_code = 0
if status != ensure_exit_code:
raise Exception('Process executed with non-0 exit code %d: %s' %
(status, args))
return status
def run_process(self, args=None, cwd=None, append_env=None,
explicit_env=None, log_name=None, log_level=logging.INFO,
line_handler=None, require_unix_environment=False,
ensure_exit_code=0, ignore_children=False, pass_thru=False):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
ensure_exit_code is used to ensure the exit code of a process matches
what is expected. If it is an integer, we raise an Exception if the
exit code does not match this value. If it is True, we ensure the exit
code is 0. If it is False, we don't perform any exit code validation.
pass_thru is a special execution mode where the child process inherits
this process's standard file handles (stdin, stdout, stderr) as well as
additional file descriptors. It should be used for interactive processes
where buffering from mozprocess could be an issue. pass_thru does not
use mozprocess. Therefore, arguments like log_name, line_handler,
and ignore_children have no effect.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args))
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.rstrip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
self.log(logging.DEBUG, 'process', {'env': use_env}, 'Environment: {env}')
# There is a bug in subprocess where it doesn't like unicode types in
# environment variables. Here, ensure all unicode are converted to
# binary. utf-8 is our globally assumed default. If the caller doesn't
# want UTF-8, they shouldn't pass in a unicode instance.
normalized_env = {}
for k, v in use_env.items():
if isinstance(k, unicode):
k = k.encode('utf-8', 'strict')
if isinstance(v, unicode):
v = v.encode('utf-8', 'strict')
normalized_env[k] = v
use_env = normalized_env
if pass_thru:
proc = subprocess.Popen(args, cwd=cwd, env=use_env)
status = None
# Leave it to the subprocess to handle Ctrl+C. If it terminates as
# a result of Ctrl+C, proc.wait() will return a status code, and,
# we get out of the loop. If it doesn't, like e.g. gdb, we continue
# waiting.
while status is None:
try:
status = proc.wait()
except KeyboardInterrupt:
pass
else:
p = ProcessHandlerMixin(args, cwd=cwd, env=use_env,
processOutputLine=[handleLine], universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = p.wait()
if ensure_exit_code is False:
return status
if ensure_exit_code is True:
ensure_exit_code = 0
if status != ensure_exit_code:
raise Exception('Process executed with non-0 exit code: %s' % args)
return status
def run_process(
self,
args=None,
cwd=None,
append_env=None,
explicit_env=None,
log_name=None,
log_level=logging.INFO,
line_handler=None,
require_unix_environment=False,
ensure_exit_code=0,
ignore_children=False,
pass_thru=False,
):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
ensure_exit_code is used to ensure the exit code of a process matches
what is expected. If it is an integer, we raise an Exception if the
exit code does not match this value. If it is True, we ensure the exit
code is 0. If it is False, we don't perform any exit code validation.
pass_thru is a special execution mode where the child process inherits
this process's standard file handles (stdin, stdout, stderr) as well as
additional file descriptors. It should be used for interactive processes
where buffering from mozprocess could be an issue. pass_thru does not
use mozprocess. Therefore, arguments like log_name, line_handler,
and ignore_children have no effect.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, "new_process", {"args": args}, " ".join(args))
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or "utf-8", "replace")
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {"line": line.strip()}, "{line}")
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
self.log(logging.DEBUG, "process", {"env": use_env}, "Environment: {env}")
if pass_thru:
status = subprocess.call(args, cwd=cwd, env=use_env)
else:
p = ProcessHandlerMixin(
args,
cwd=cwd,
env=use_env,
processOutputLine=[handleLine],
universal_newlines=True,
ignore_children=ignore_children,
)
p.run()
p.processOutput()
status = p.wait()
if ensure_exit_code is False:
return status
if ensure_exit_code is True:
ensure_exit_code = 0
if status != ensure_exit_code:
raise Exception("Process executed with non-0 exit code: %s" % args)
return status
def InvokeClWithDependencyGeneration(cmdline):
target = ""
# Figure out what the target is
for arg in cmdline:
if arg.startswith("-Fo"):
target = arg[3:]
break
if target is None:
print >>sys.stderr, "No target set"
return 1
# Assume the source file is the last argument
source = cmdline[-1]
assert not source.startswith('-')
# The deps target lives here
depstarget = os.path.basename(target) + ".pp"
cmdline += ['-showIncludes']
mk = Makefile()
rule = mk.create_rule([target])
rule.add_dependencies([normcase(source)])
def on_line(line):
# cl -showIncludes prefixes every header with "Note: including file:"
# and an indentation corresponding to the depth (which we don't need)
if line.startswith(CL_INCLUDES_PREFIX):
dep = line[len(CL_INCLUDES_PREFIX):].strip()
# We can't handle pathes with spaces properly in mddepend.pl, but
# we can assume that anything in a path with spaces is a system
# header and throw it away.
dep = normcase(dep)
if ' ' not in dep:
rule.add_dependencies([dep])
else:
# Make sure we preserve the relevant output from cl. mozprocess
# swallows the newline delimiter, so we need to re-add it.
sys.stdout.write(line)
sys.stdout.write('\n')
# We need to ignore children because MSVC can fire up a background process
# during compilation. This process is cleaned up on its own. If we kill it,
# we can run into weird compilation issues.
p = ProcessHandlerMixin(cmdline, processOutputLine=[on_line],
ignore_children=True)
p.run()
p.processOutput()
ret = p.wait()
if ret != 0 or target == "":
# p.wait() returns a long. Somehow sys.exit(long(0)) is like
# sys.exit(1). Don't ask why.
return int(ret)
depsdir = os.path.normpath(os.path.join(os.curdir, ".deps"))
depstarget = os.path.join(depsdir, depstarget)
if not os.path.isdir(depsdir):
try:
os.makedirs(depsdir)
except OSError:
pass # This suppresses the error we get when the dir exists, at the
# cost of masking failure to create the directory. We'll just
# die on the next line though, so it's not that much of a loss.
with open(depstarget, "w") as f:
mk.dump(f)
return 0
def run_process(self,
args=None,
cwd=None,
append_env=None,
explicit_env=None,
log_name=None,
log_level=logging.INFO,
line_handler=None,
require_unix_environment=False,
ignore_errors=False,
ignore_children=False):
"""Runs a single process to completion.
Takes a list of arguments to run where the first item is the
executable. Runs the command in the specified directory and
with optional environment variables.
append_env -- Dict of environment variables to append to the current
set of environment variables.
explicit_env -- Dict of environment variables to set for the new
process. Any existing environment variables will be ignored.
require_unix_environment if True will ensure the command is executed
within a UNIX environment. Basically, if we are on Windows, it will
execute the command via an appropriate UNIX-like shell.
ignore_children is proxied to mozprocess's ignore_children.
"""
args = self._normalize_command(args, require_unix_environment)
self.log(logging.INFO, 'new_process', {'args': args}, ' '.join(args))
def handleLine(line):
# Converts str to unicode on Python 2 and bytes to str on Python 3.
if isinstance(line, bytes):
line = line.decode(sys.stdout.encoding or 'utf-8', 'replace')
if line_handler:
line_handler(line)
if not log_name:
return
self.log(log_level, log_name, {'line': line.strip()}, '{line}')
use_env = {}
if explicit_env:
use_env = explicit_env
else:
use_env.update(os.environ)
if append_env:
use_env.update(append_env)
self.log(logging.DEBUG, 'process', {'env': use_env},
'Environment: {env}')
p = ProcessHandlerMixin(args,
cwd=cwd,
env=use_env,
processOutputLine=[handleLine],
universal_newlines=True,
ignore_children=ignore_children)
p.run()
p.processOutput()
status = p.wait()
if status != 0 and not ignore_errors:
raise Exception('Process executed with non-0 exit code: %s' % args)