def check_output(*popenargs, **kwargs):
"""Call a process and check result code.
This is for Python 2.6 compatibility, which doesn't have that in its
standard library.
Note: We use same name as in Python stdlib, violating our rules to
make it more recognizable what this does.
"""
if "stdout" in kwargs:
raise ValueError("stdout argument not allowed, it will be overridden.")
if "stderr" not in kwargs:
kwargs["stderr"] = subprocess.PIPE
process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
output, stderr = process.communicate()
retcode = process.poll()
if retcode:
cmd = kwargs.get("args")
if cmd is None:
cmd = popenargs[0]
raise NuitkaCalledProcessError(retcode, cmd, output=output, stderr=stderr)
return output
def executeToolChecked(logger, command, absence_message, stderr_filter=None):
"""Execute external tool, checking for success and no error outputs, returning result."""
command = list(command)
tool = command[0]
if not isExecutableCommand(tool):
logger.sysexit(absence_message)
# Allow to avoid repeated scans in PATH for the tool.
command[0] = getExecutablePath(tool)
process = subprocess.Popen(
command,
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False,
)
stdout, stderr = process.communicate()
result = process.poll()
if stderr_filter is not None:
stderr = stderr_filter(stderr)
if result != 0:
logger.sysexit("Error, call to %r failed: %s -> %s." % (tool, command, stderr))
elif stderr:
logger.sysexit(
"Error, call to %r gave warnings: %s -> %s." % (tool, command, stderr)
)
return stdout
def executeProcess(command, env=None, stdin=False, shell=False, external_cwd=False):
if not env:
env = os.environ
# Note: Empty string should also be allowed for stdin, therefore check for
# default "False" and "None" precisely.
process = subprocess.Popen(
command,
stdin=subprocess.PIPE if stdin not in (False, None) else getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=shell,
# On Windows, closing file descriptions is now working with capturing outputs.
close_fds=not isWin32Windows(),
env=env,
# For tools that want short paths to work.
cwd=getExternalUsePath(os.getcwd()) if external_cwd else None,
)
if stdin is True:
process_input = None
elif stdin is not False:
process_input = stdin
else:
process_input = None
stdout, stderr = process.communicate(input=process_input)
exit_code = process.wait()
return stdout, stderr, exit_code
def updateWorkingFile(path, orig_object_hash, new_object_hash):
patch = check_output(
["git", "diff", "--no-color", orig_object_hash, new_object_hash])
git_path = path.replace(os.path.sep, "/").encode("utf8")
def updateLine(line):
if line.startswith(b"diff --git"):
line = b"diff --git a/%s b/%s" % (git_path, git_path)
elif line.startswith(b"--- a/"):
line = b"--- a/" + git_path
elif line.startswith(b"+++ b/"):
line = b"+++ b/" + git_path
return line
# Substitute object hashes in patch header with path to working tree file
patch = b"\n".join(updateLine(line) for line in patch.splitlines()) + b"\n"
# Cannot use executeProcess, because we supply input, and we cannot
# use context managers, because not all Python versions have it that
# way, xpylintx: disable=consider-using-with
apply_patch = subprocess.Popen(
["git", "apply", "-"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
output, err = apply_patch.communicate(input=patch)
success = apply_patch.returncode == 0
if not success:
# TODO: In case of failure, do we need to abort, or what do we do.
if output:
my_print(output, style="yellow")
if err:
my_print(err, style="yellow")
return success
def getRuntimeTraceOfLoadedFiles(logger, command, required=False):
"""Returns the files loaded when executing a binary."""
# This will make a crazy amount of work,
# pylint: disable=I0021,too-many-branches,too-many-locals,too-many-statements
path = command[0]
if not os.path.exists(path):
logger.sysexit("Error, cannot find %r (%r)." % (path, os.path.abspath(path)))
result = []
if os.name == "posix":
if getOS() in ("Darwin", "FreeBSD"):
if not isExecutableCommand("dtruss"):
logger.sysexit(
"""\
Error, needs 'dtruss' on your system to scan used libraries."""
)
if not isExecutableCommand("sudo"):
logger.sysexit(
"""\
Error, needs 'sudo' on your system to scan used libraries."""
)
args = ("sudo", "dtruss", "-t", "open", os.path.abspath(path))
else:
if not isExecutableCommand("strace"):
logger.sysexit(
"""\
Error, needs 'strace' on your system to scan used libraries."""
)
args = (
"strace",
"-e",
"file",
"-s4096", # Some paths are truncated in output otherwise
os.path.abspath(path),
)
# Ensure executable is not polluted with third party stuff,
# tests may fail otherwise due to unexpected libs being loaded
with withEnvironmentVarOverridden("LD_PRELOAD", None):
tracing_command = args[0] if args[0] != "sudo" else args[1]
process = subprocess.Popen(
args=args, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
_stdout_strace, stderr_strace = process.communicate()
exit_strace = process.returncode
if exit_strace != 0:
if str is not bytes:
stderr_strace = stderr_strace.decode("utf8")
logger.warning(stderr_strace)
logger.sysexit("Failed to run %r." % tracing_command)
with open(path + ".strace", "wb") as f:
f.write(stderr_strace)
for line in stderr_strace.split(b"\n"):
if process.returncode != 0:
logger.my_print(line)
if not line:
continue
# Don't consider files not found. The "site" module checks lots
# of things.
if b"ENOENT" in line:
continue
if line.startswith(b"stat(") and b"S_IFDIR" in line:
continue
result.extend(
os.path.abspath(match)
for match in re.findall(b'"(.*?)(?:\\\\0)?"', line)
)
if str is not bytes:
result = [s.decode("utf8") for s in result]
elif os.name == "nt":
command = (
getDependsExePath(),
"-c", # Console mode
"-ot%s" % path + ".depends",
"-f1",
"-pb",
"-pa1", # Turn on all profiling options.
"-ps1", # Simulate ShellExecute with app dirs in PATH.
"-pp1", # Do not long DllMain calls.
"-po1", # Log DllMain call for all other messages.
"-ph1", # Hook the process.
"-pl1", # Log LoadLibrary calls.
"-pt1", # Thread information.
"-pe1", # First chance exceptions.
"-pg1", # Log GetProcAddress calls.
"-pf1", # Use full paths.
"-pc1", # Profile child processes.
) + tuple(command)
# TODO: Move the handling of this into nuitka.tools.Execution module methods.
try:
callProcess(command, timeout=5 * 60)
except Exception as e: # Catch all the things, pylint: disable=broad-except
if e.__class__.__name__ == "TimeoutExpired":
if required:
logger.sysexit(
"Timeout encountered when running dependency walker."
)
logger.warning("Timeout encountered when running dependency walker.")
return []
else:
raise
result = parseDependsExeOutput(path + ".depends")
os.unlink(path + ".depends")
result = list(sorted(set(result)))
return result