def _runCPgoBinary():
# Note: For exit codes, we do not insist on anything yet, maybe we could point it out
# or ask people to make scripts that buffer these kinds of errors, and take an error
# instead as a serious failure.
general.info("Running created binary to produce C level PGO information:",
style="blue")
if _wasMsvcMode():
msvc_pgc_filename = _deleteMsvcPGOFiles(pgo_mode="generate")
with withEnvironmentVarOverridden(
"PATH",
getSconsReportValue(
source_dir=OutputDirectories.getSourceDirectoryPath(),
key="PATH"),
):
_exit_code = _runPgoBinary()
pgo_data_collected = os.path.exists(msvc_pgc_filename)
else:
_exit_code = _runPgoBinary()
gcc_constants_pgo_filename = os.path.join(
OutputDirectories.getSourceDirectoryPath(), "__constants.gcda")
pgo_data_collected = os.path.exists(gcc_constants_pgo_filename)
if not pgo_data_collected:
general.sysexit(
"Error, no PGO information produced, did the created binary run at all?"
)
general.info("Successfully collected C level PGO information.",
style="blue")
def wrapCommandForDebuggerForExec(*args):
"""Wrap a command for system debugger to call exec
Args:
args: (list of str) args for call to be debugged
Returns:
args tuple with debugger command inserted
Notes:
Currently only gdb and lldb are supported, but adding more
debuggers would be very welcome.
"""
gdb_path = getExecutablePath("gdb")
lldb_path = getExecutablePath("lldb")
if gdb_path is None and lldb_path is None:
general.sysexit("Error, no 'gdb' or 'lldb' binary found in path.")
if gdb_path is not None:
args = (gdb_path, "gdb", "-ex=run", "-ex=where", "-ex=quit",
"--args") + args
else:
args = (lldb_path, "lldb", "-o", "run", "-o", "bt", "-o", "quit",
"--") + args
return args
def compileTree():
source_dir = OutputDirectories.getSourceDirectoryPath()
general.info("Completed Python level compilation and optimization.")
if not Options.shallOnlyExecCCompilerCall():
general.info("Generating source code for C backend compiler.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before generating C code: {memory}:".
format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()))
# Now build the target language code for the whole tree.
makeSourceDirectory()
bytecode_accessor = ConstantAccessor(data_filename="__bytecode.const",
top_level_name="bytecode_data")
# This should take all bytecode values, even ones needed for frozen or
# not produce anything.
loader_code = LoaderCodes.getMetapathLoaderBodyCode(bytecode_accessor)
writeSourceCode(filename=os.path.join(source_dir, "__loader.c"),
source_code=loader_code)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
general.sysexit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()))
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.is_debug:
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
general.info(
"Running data composer tool for optimal constant value handling.")
# TODO: On Windows, we could run this in parallel to Scons, on Linux we need it
# for linking.
runDataComposer(source_dir)
general.info("Running C level backend compilation via Scons.")
# Run the Scons to build things.
result, options = runSconsBackend(quiet=not Options.isShowScons())
return result, options
def checkPythonVersionFromCode(source_code):
# There is a lot of cases to consider, pylint: disable=too-many-branches
shebang = getShebangFromSource(source_code)
if shebang is not None:
binary, _args = parseShebang(shebang)
if getOS() != "Windows":
try:
if os.path.samefile(sys.executable, binary):
return True
except OSError: # Might not exist
pass
basename = os.path.basename(binary)
# Not sure if we should do that.
if basename == "python":
result = python_version < 0x300
elif basename == "python3":
result = python_version > 0x300
elif basename == "python2":
result = python_version < 0x300
elif basename == "python2.7":
result = python_version < 0x300
elif basename == "python2.6":
result = python_version < 0x270
elif basename == "python3.2":
result = 0x330 > python_version >= 0x300
elif basename == "python3.3":
result = 0x340 > python_version >= 0x330
elif basename == "python3.4":
result = 0x350 > python_version >= 0x340
elif basename == "python3.5":
result = 0x360 > python_version >= 0x350
elif basename == "python3.6":
result = 0x370 > python_version >= 0x360
elif basename == "python3.7":
result = 0x380 > python_version >= 0x370
elif basename == "python3.8":
result = 0x390 > python_version >= 0x380
elif basename == "python3.9":
result = 0x3A0 > python_version >= 0x390
else:
result = None
if result is False:
general.sysexit("""\
The program you compiled wants to be run with: %s.
Nuitka is currently running with Python version '%s', which seems to not
match that. Nuitka cannot guess the Python version of your source code. You
therefore might want to specify: '%s -m nuitka'.
That will make use the correct Python version for Nuitka.
""" % (shebang, python_version_str, binary))
def _runPgoBinary():
pgo_executable = OutputDirectories.getPgoRunExecutable()
if not os.path.isfile(pgo_executable):
general.sysexit("Error, failed to produce PGO binary '%s'" % pgo_executable)
return callProcess(
[getExternalUsePath(pgo_executable)] + Options.getPgoArgs(),
shell=False,
)
def startGraph():
# We maintain this globally to make it accessible, pylint: disable=global-statement
global graph
if Options.shallCreateGraph():
try:
from pygraphviz import AGraph # pylint: disable=I0021,import-error
graph = AGraph(name="Optimization", directed=True)
graph.layout()
except ImportError:
general.sysexit(
"Cannot import pygraphviz module, no graphing capability.")
def check_call(*popenargs, **kwargs):
"""Call a process and check result code.
Note: This catches the error, and makes it nicer, and an error
exit. So this is for tooling only.
Note: We use same name as in Python stdlib, violating our rules to
make it more recognizable what this does.
"""
try:
subprocess.check_call(*popenargs, **kwargs)
except OSError:
general.sysexit("Error, failed to execute '%s'. Is it installed?" %
popenargs[0])
def _runPythonPgoBinary():
# Note: For exit codes, we do not insist on anything yet, maybe we could point it out
# or ask people to make scripts that buffer these kinds of errors, and take an error
# instead as a serious failure.
pgo_filename = OutputDirectories.getPgoRunInputFilename()
with withEnvironmentVarOverridden("NUITKA_PGO_OUTPUT", pgo_filename):
_exit_code = _runPgoBinary()
if not os.path.exists(pgo_filename):
general.sysexit(
"Error, no Python PGO information produced, did the created binary run at all?"
)
return pgo_filename
def wrapCommandForDebuggerForExec(*args):
"""Wrap a command for system debugger to call exec
Args:
args: (list of str) args for call to be debugged
Returns:
args tuple with debugger command inserted
Notes:
Currently only gdb and lldb are supported, but adding more
debuggers would be very welcome.
"""
gdb_path = getExecutablePath("gdb")
# Windows extra ball, attempt the downloaded one.
if isWin32Windows() and gdb_path is None:
from nuitka.Options import assumeYesForDownloads
mingw64_gcc_path = getCachedDownloadedMinGW64(
target_arch=getArchitecture(),
assume_yes_for_downloads=assumeYesForDownloads(),
)
with withEnvironmentPathAdded("PATH", os.path.dirname(mingw64_gcc_path)):
lldb_path = getExecutablePath("lldb")
if gdb_path is None and lldb_path is None:
lldb_path = getExecutablePath("lldb")
if lldb_path is None:
general.sysexit("Error, no 'gdb' or 'lldb' binary found in path.")
if gdb_path is not None:
args = (gdb_path, "gdb", "-ex=run", "-ex=where", "-ex=quit", "--args") + args
else:
args = (lldb_path, "lldb", "-o", "run", "-o", "bt", "-o", "quit", "--") + args
return args
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 0x300:
command = command.encode("utf8")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args=[sys.executable, "-s", "-S", "-v", tmp_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dict(os.environ, PYTHONIOENCODING="utf_8"),
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf-8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300:
if module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "shlib", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "shlib", filename))
for module_name, _prio, kind, filename in sorted(detections):
if kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "shlib":
_detectedShlibFile(filename=filename, module_name=module_name)
else:
assert False, kind
return result
def buildAssignmentStatementsFromDecoded(provider, kind, detail, source,
source_ref):
# This is using many variable names on purpose, so as to give names to the
# unpacked detail values, and has many branches due to the many cases
# dealt with and it is return driven.
# pylint: disable=too-many-branches,too-many-locals,too-many-return-statements,too-many-statements
if kind == "Name":
if detail in ("_inject_c_code",
"_inject_c_decl") and isExperimental("c-code-injection"):
if not source.isExpressionConstantStrRef():
general.sysexit(
"Error, value assigned to '%s' not be constant str" %
detail)
if detail == "_inject_c_code":
return StatementInjectCCode(
c_code=source.getCompileTimeConstant(),
source_ref=source_ref)
else:
return StatementInjectCDecl(
c_code=source.getCompileTimeConstant(),
source_ref=source_ref)
return StatementAssignmentVariableName(
provider=provider,
variable_name=detail,
source=source,
source_ref=source_ref,
)
elif kind == "Attribute":
lookup_source, attribute_name = detail
return StatementAssignmentAttribute(
expression=lookup_source,
attribute_name=mangleName(attribute_name, provider),
source=source,
source_ref=source_ref,
)
elif kind == "Subscript":
subscribed, subscript = detail
return StatementAssignmentSubscript(
subscribed=subscribed,
subscript=subscript,
source=source,
source_ref=source_ref,
)
elif kind == "Slice":
lookup_source, lower, upper = detail
# For Python3 there is no slicing operation, this is always done
# with subscript using a slice object. For Python2, it is only done
# if no "step" is provided.
use_sliceobj = python_version >= 0x300
if use_sliceobj:
return StatementAssignmentSubscript(
subscribed=lookup_source,
source=source,
subscript=makeExpressionBuiltinSlice(start=lower,
stop=upper,
step=None,
source_ref=source_ref),
source_ref=source_ref,
)
else:
return StatementAssignmentSlice(
expression=lookup_source,
lower=lower,
upper=upper,
source=source,
source_ref=source_ref,
)
elif kind == "Tuple":
temp_scope = provider.allocateTempScope("tuple_unpack")
source_iter_var = provider.allocateTempVariable(temp_scope=temp_scope,
name="source_iter")
element_vars = [
provider.allocateTempVariable(temp_scope=temp_scope,
name="element_%d" %
(element_index + 1))
for element_index in range(len(detail))
]
starred_list_var = None
starred_index = None
statements = []
for element_index, element in enumerate(detail):
if element[0] == "Starred":
if starred_index is not None:
raiseSyntaxError(
"two starred expressions in assignment"
if python_version < 0x390 else
"multiple starred expressions in assignment",
source_ref.atColumnNumber(0),
)
starred_index = element_index
for element_index, element in enumerate(detail):
element_var = element_vars[element_index]
if starred_list_var is not None:
statements.insert(
starred_index + 1,
StatementAssignmentVariable(
variable=element_var,
source=ExpressionListOperationPop(
list_arg=ExpressionTempVariableRef(
variable=starred_list_var,
source_ref=source_ref),
source_ref=source_ref,
),
source_ref=source_ref,
),
)
elif element[0] != "Starred":
statements.append(
StatementAssignmentVariable(
variable=element_var,
source=ExpressionSpecialUnpack(
value=ExpressionTempVariableRef(
variable=source_iter_var,
source_ref=source_ref),
count=element_index + 1,
expected=starred_index or len(detail),
starred=starred_index is not None,
source_ref=source_ref,
),
source_ref=source_ref,
))
else:
assert starred_index == element_index
starred_list_var = element_var
statements.append(
StatementAssignmentVariable(
variable=element_var,
source=ExpressionBuiltinList(
value=ExpressionTempVariableRef(
variable=source_iter_var,
source_ref=source_ref),
source_ref=source_ref,
),
source_ref=source_ref,
))
if starred_list_var is None:
statements.append(
StatementSpecialUnpackCheck(
iterator=ExpressionTempVariableRef(
variable=source_iter_var, source_ref=source_ref),
count=len(detail),
source_ref=source_ref,
))
else:
statements.insert(
starred_index + 1,
makeStatementConditional(
condition=makeComparisonExpression(
comparator="Lt",
left=ExpressionBuiltinLen(
value=ExpressionTempVariableRef(
variable=starred_list_var,
source_ref=source_ref),
source_ref=source_ref,
),
right=makeConstantRefNode(
constant=len(statements) - starred_index - 1,
source_ref=source_ref,
),
source_ref=source_ref,
),
yes_branch=makeRaiseExceptionExpressionFromTemplate(
exception_type="ValueError",
template="""\
not enough values to unpack (expected at least %d, got %%d)""" %
(len(statements) - 1),
template_args=makeBinaryOperationNode(
operator="Add",
left=ExpressionBuiltinLen(
value=ExpressionTempVariableRef(
variable=starred_list_var,
source_ref=source_ref),
source_ref=source_ref,
),
right=makeConstantRefNode(constant=starred_index,
source_ref=source_ref),
source_ref=source_ref,
),
source_ref=source_ref,
).asStatement(),
no_branch=None,
source_ref=source_ref,
),
)
if python_version >= 0x370:
iter_creation_class = ExpressionBuiltinIterForUnpack
else:
iter_creation_class = ExpressionBuiltinIter1
statements = [
StatementAssignmentVariable(
variable=source_iter_var,
source=iter_creation_class(value=source,
source_ref=source_ref),
source_ref=source_ref,
),
makeTryFinallyStatement(
provider=provider,
tried=statements,
final=(StatementReleaseVariable(variable=source_iter_var,
source_ref=source_ref), ),
source_ref=source_ref,
),
]
# When all is done, copy over to the actual assignment targets, starred
# or not makes no difference here anymore.
for element_index, element in enumerate(detail):
if element[0] == "Starred":
element = element[1]
element_var = element_vars[element_index]
statements.append(
buildAssignmentStatementsFromDecoded(
provider=provider,
kind=element[0],
detail=element[1],
source=ExpressionTempVariableRef(variable=element_var,
source_ref=source_ref),
source_ref=source_ref,
))
# Need to release temporary variables right after successful
# usage.
statements.append(
StatementDelVariable(variable=element_var,
tolerant=True,
source_ref=source_ref))
final_statements = []
for element_var in element_vars:
final_statements.append(
StatementReleaseVariable(variable=element_var,
source_ref=source_ref))
return makeTryFinallyStatement(
provider=provider,
tried=statements,
final=final_statements,
source_ref=source_ref,
)
elif kind == "Starred":
raiseSyntaxError(
"starred assignment target must be in a list or tuple",
source_ref.atColumnNumber(0),
)
else:
assert False, (kind, source_ref, detail)
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
if str is not bytes:
command = command.encode("utf8")
_stdout, stderr, exit_code = executeProcess(
command=(
sys.executable,
"-s",
"-S",
"-v",
"-c",
"import sys;exec(sys.stdin.read())",
),
stdin=command,
env=dict(os.environ, PYTHONIOENCODING="utf-8"),
)
assert type(stderr) is bytes
# Don't let errors here go unnoticed.
if exit_code != 0:
# An error by the user pressing CTRL-C should not lead to the below output.
if b"KeyboardInterrupt" in stderr:
general.sysexit("Pressed CTRL-C while detecting early imports.")
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"from" and python_version < 0x300:
filename = parts[1][len(b"from "):]
if str is not bytes: # For consistency, and maybe later reuse
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
else:
assert False
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300 and module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "extension", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "extension", filename))
for module_name, _priority, kind, filename in sorted(detections):
if isStandardLibraryNoAutoInclusionModule(module_name):
continue
if kind == "extension":
_detectedExtensionModule(
filename=filename,
module_name=module_name,
result=result,
technical=technical,
)
elif kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
else:
assert False, kind
return result
