def _runCPgoBinary(): # Note: For exit codes, we do not insist on anything yet, maybe we could point it out # or ask people to make scripts that buffer these kinds of errors, and take an error # instead as a serious failure. general.info("Running created binary to produce C level PGO information:", style="blue") if _wasMsvcMode(): msvc_pgc_filename = _deleteMsvcPGOFiles(pgo_mode="generate") with withEnvironmentVarOverridden( "PATH", getSconsReportValue( source_dir=OutputDirectories.getSourceDirectoryPath(), key="PATH"), ): _exit_code = _runPgoBinary() pgo_data_collected = os.path.exists(msvc_pgc_filename) else: _exit_code = _runPgoBinary() gcc_constants_pgo_filename = os.path.join( OutputDirectories.getSourceDirectoryPath(), "__constants.gcda") pgo_data_collected = os.path.exists(gcc_constants_pgo_filename) if not pgo_data_collected: general.sysexit( "Error, no PGO information produced, did the created binary run at all?" ) general.info("Successfully collected C level PGO information.", style="blue")
def wrapCommandForDebuggerForExec(*args): """Wrap a command for system debugger to call exec Args: args: (list of str) args for call to be debugged Returns: args tuple with debugger command inserted Notes: Currently only gdb and lldb are supported, but adding more debuggers would be very welcome. """ gdb_path = getExecutablePath("gdb") lldb_path = getExecutablePath("lldb") if gdb_path is None and lldb_path is None: general.sysexit("Error, no 'gdb' or 'lldb' binary found in path.") if gdb_path is not None: args = (gdb_path, "gdb", "-ex=run", "-ex=where", "-ex=quit", "--args") + args else: args = (lldb_path, "lldb", "-o", "run", "-o", "bt", "-o", "quit", "--") + args return args
def compileTree(): source_dir = OutputDirectories.getSourceDirectoryPath() general.info("Completed Python level compilation and optimization.") if not Options.shallOnlyExecCCompilerCall(): general.info("Generating source code for C backend compiler.") if Options.isShowProgress() or Options.isShowMemory(): general.info( "Total memory usage before generating C code: {memory}:". format( memory=MemoryUsage.getHumanReadableProcessMemoryUsage())) # Now build the target language code for the whole tree. makeSourceDirectory() bytecode_accessor = ConstantAccessor(data_filename="__bytecode.const", top_level_name="bytecode_data") # This should take all bytecode values, even ones needed for frozen or # not produce anything. loader_code = LoaderCodes.getMetapathLoaderBodyCode(bytecode_accessor) writeSourceCode(filename=os.path.join(source_dir, "__loader.c"), source_code=loader_code) else: source_dir = OutputDirectories.getSourceDirectoryPath() if not os.path.isfile(os.path.join(source_dir, "__helpers.h")): general.sysexit("Error, no previous build directory exists.") if Options.isShowProgress() or Options.isShowMemory(): general.info( "Total memory usage before running scons: {memory}:".format( memory=MemoryUsage.getHumanReadableProcessMemoryUsage())) if Options.isShowMemory(): InstanceCounters.printStats() if Options.is_debug: Reports.doMissingOptimizationReport() if Options.shallNotDoExecCCompilerCall(): return True, {} general.info( "Running data composer tool for optimal constant value handling.") # TODO: On Windows, we could run this in parallel to Scons, on Linux we need it # for linking. runDataComposer(source_dir) general.info("Running C level backend compilation via Scons.") # Run the Scons to build things. result, options = runSconsBackend(quiet=not Options.isShowScons()) return result, options
def checkPythonVersionFromCode(source_code): # There is a lot of cases to consider, pylint: disable=too-many-branches shebang = getShebangFromSource(source_code) if shebang is not None: binary, _args = parseShebang(shebang) if getOS() != "Windows": try: if os.path.samefile(sys.executable, binary): return True except OSError: # Might not exist pass basename = os.path.basename(binary) # Not sure if we should do that. if basename == "python": result = python_version < 0x300 elif basename == "python3": result = python_version > 0x300 elif basename == "python2": result = python_version < 0x300 elif basename == "python2.7": result = python_version < 0x300 elif basename == "python2.6": result = python_version < 0x270 elif basename == "python3.2": result = 0x330 > python_version >= 0x300 elif basename == "python3.3": result = 0x340 > python_version >= 0x330 elif basename == "python3.4": result = 0x350 > python_version >= 0x340 elif basename == "python3.5": result = 0x360 > python_version >= 0x350 elif basename == "python3.6": result = 0x370 > python_version >= 0x360 elif basename == "python3.7": result = 0x380 > python_version >= 0x370 elif basename == "python3.8": result = 0x390 > python_version >= 0x380 elif basename == "python3.9": result = 0x3A0 > python_version >= 0x390 else: result = None if result is False: general.sysexit("""\ The program you compiled wants to be run with: %s. Nuitka is currently running with Python version '%s', which seems to not match that. Nuitka cannot guess the Python version of your source code. You therefore might want to specify: '%s -m nuitka'. That will make use the correct Python version for Nuitka. """ % (shebang, python_version_str, binary))
def _runPgoBinary(): pgo_executable = OutputDirectories.getPgoRunExecutable() if not os.path.isfile(pgo_executable): general.sysexit("Error, failed to produce PGO binary '%s'" % pgo_executable) return callProcess( [getExternalUsePath(pgo_executable)] + Options.getPgoArgs(), shell=False, )
def startGraph(): # We maintain this globally to make it accessible, pylint: disable=global-statement global graph if Options.shallCreateGraph(): try: from pygraphviz import AGraph # pylint: disable=I0021,import-error graph = AGraph(name="Optimization", directed=True) graph.layout() except ImportError: general.sysexit( "Cannot import pygraphviz module, no graphing capability.")
def check_call(*popenargs, **kwargs): """Call a process and check result code. Note: This catches the error, and makes it nicer, and an error exit. So this is for tooling only. Note: We use same name as in Python stdlib, violating our rules to make it more recognizable what this does. """ try: subprocess.check_call(*popenargs, **kwargs) except OSError: general.sysexit("Error, failed to execute '%s'. Is it installed?" % popenargs[0])
def _runPythonPgoBinary(): # Note: For exit codes, we do not insist on anything yet, maybe we could point it out # or ask people to make scripts that buffer these kinds of errors, and take an error # instead as a serious failure. pgo_filename = OutputDirectories.getPgoRunInputFilename() with withEnvironmentVarOverridden("NUITKA_PGO_OUTPUT", pgo_filename): _exit_code = _runPgoBinary() if not os.path.exists(pgo_filename): general.sysexit( "Error, no Python PGO information produced, did the created binary run at all?" ) return pgo_filename
def wrapCommandForDebuggerForExec(*args): """Wrap a command for system debugger to call exec Args: args: (list of str) args for call to be debugged Returns: args tuple with debugger command inserted Notes: Currently only gdb and lldb are supported, but adding more debuggers would be very welcome. """ gdb_path = getExecutablePath("gdb") # Windows extra ball, attempt the downloaded one. if isWin32Windows() and gdb_path is None: from nuitka.Options import assumeYesForDownloads mingw64_gcc_path = getCachedDownloadedMinGW64( target_arch=getArchitecture(), assume_yes_for_downloads=assumeYesForDownloads(), ) with withEnvironmentPathAdded("PATH", os.path.dirname(mingw64_gcc_path)): lldb_path = getExecutablePath("lldb") if gdb_path is None and lldb_path is None: lldb_path = getExecutablePath("lldb") if lldb_path is None: general.sysexit("Error, no 'gdb' or 'lldb' binary found in path.") if gdb_path is not None: args = (gdb_path, "gdb", "-ex=run", "-ex=where", "-ex=quit", "--args") + args else: args = (lldb_path, "lldb", "-o", "run", "-o", "bt", "-o", "quit", "--") + args return args
def _detectImports(command, user_provided, technical): # This is pretty complicated stuff, with variants to deal with. # pylint: disable=too-many-branches,too-many-locals,too-many-statements # Print statements for stuff to show, the modules loaded. if python_version >= 0x300: command += """ print("\\n".join(sorted( "import %s # sourcefile %s" % (module.__name__, module.__file__) for module in sys.modules.values() if getattr(module, "__file__", None) not in (None, "<frozen>" ))), file = sys.stderr)""" reduced_path = [ path_element for path_element in sys.path if not areSamePaths(path_element, ".") if not areSamePaths( path_element, os.path.dirname(sys.modules["__main__"].__file__)) ] # Make sure the right import path (the one Nuitka binary is running with) # is used. command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" % repr(reduced_path)) + command import tempfile tmp_file, tmp_filename = tempfile.mkstemp() try: if python_version >= 0x300: command = command.encode("utf8") os.write(tmp_file, command) os.close(tmp_file) process = subprocess.Popen( args=[sys.executable, "-s", "-S", "-v", tmp_filename], stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=dict(os.environ, PYTHONIOENCODING="utf_8"), ) _stdout, stderr = process.communicate() finally: os.unlink(tmp_filename) # Don't let errors here go unnoticed. if process.returncode != 0: general.warning( "There is a problem with detecting imports, CPython said:") for line in stderr.split(b"\n"): printError(line) general.sysexit("Error, please report the issue with above output.") result = [] detections = [] for line in stderr.replace(b"\r", b"").split(b"\n"): if line.startswith(b"import "): # print(line) parts = line.split(b" # ", 2) module_name = parts[0].split(b" ", 2)[1] origin = parts[1].split()[0] if python_version >= 0x300: module_name = module_name.decode("utf-8") module_name = ModuleName(module_name) if origin == b"precompiled": # This is a ".pyc" file that was imported, even before we have a # chance to do anything, we need to preserve it. filename = parts[1][len(b"precompiled from "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 3, "precompiled", filename)) elif origin == b"sourcefile": filename = parts[1][len(b"sourcefile "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) elif filename.endswith(".pyc"): detections.append( (module_name, 3, "precompiled", filename)) elif not filename.endswith("<frozen>"): # Python3 started lying in "__name__" for the "_decimal" # calls itself "decimal", which then is wrong and also # clashes with "decimal" proper if python_version >= 0x300: if module_name == "decimal": module_name = ModuleName("_decimal") detections.append((module_name, 2, "shlib", filename)) elif origin == b"dynamically": # Shared library in early load, happens on RPM based systems and # or self compiled Python installations. filename = parts[1][len(b"dynamically loaded from "):] if python_version >= 0x300: filename = filename.decode("utf-8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 1, "shlib", filename)) for module_name, _prio, kind, filename in sorted(detections): if kind == "precompiled": _detectedPrecompiledFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "sourcefile": _detectedSourceFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "shlib": _detectedShlibFile(filename=filename, module_name=module_name) else: assert False, kind return result
def buildAssignmentStatementsFromDecoded(provider, kind, detail, source, source_ref): # This is using many variable names on purpose, so as to give names to the # unpacked detail values, and has many branches due to the many cases # dealt with and it is return driven. # pylint: disable=too-many-branches,too-many-locals,too-many-return-statements,too-many-statements if kind == "Name": if detail in ("_inject_c_code", "_inject_c_decl") and isExperimental("c-code-injection"): if not source.isExpressionConstantStrRef(): general.sysexit( "Error, value assigned to '%s' not be constant str" % detail) if detail == "_inject_c_code": return StatementInjectCCode( c_code=source.getCompileTimeConstant(), source_ref=source_ref) else: return StatementInjectCDecl( c_code=source.getCompileTimeConstant(), source_ref=source_ref) return StatementAssignmentVariableName( provider=provider, variable_name=detail, source=source, source_ref=source_ref, ) elif kind == "Attribute": lookup_source, attribute_name = detail return StatementAssignmentAttribute( expression=lookup_source, attribute_name=mangleName(attribute_name, provider), source=source, source_ref=source_ref, ) elif kind == "Subscript": subscribed, subscript = detail return StatementAssignmentSubscript( subscribed=subscribed, subscript=subscript, source=source, source_ref=source_ref, ) elif kind == "Slice": lookup_source, lower, upper = detail # For Python3 there is no slicing operation, this is always done # with subscript using a slice object. For Python2, it is only done # if no "step" is provided. use_sliceobj = python_version >= 0x300 if use_sliceobj: return StatementAssignmentSubscript( subscribed=lookup_source, source=source, subscript=makeExpressionBuiltinSlice(start=lower, stop=upper, step=None, source_ref=source_ref), source_ref=source_ref, ) else: return StatementAssignmentSlice( expression=lookup_source, lower=lower, upper=upper, source=source, source_ref=source_ref, ) elif kind == "Tuple": temp_scope = provider.allocateTempScope("tuple_unpack") source_iter_var = provider.allocateTempVariable(temp_scope=temp_scope, name="source_iter") element_vars = [ provider.allocateTempVariable(temp_scope=temp_scope, name="element_%d" % (element_index + 1)) for element_index in range(len(detail)) ] starred_list_var = None starred_index = None statements = [] for element_index, element in enumerate(detail): if element[0] == "Starred": if starred_index is not None: raiseSyntaxError( "two starred expressions in assignment" if python_version < 0x390 else "multiple starred expressions in assignment", source_ref.atColumnNumber(0), ) starred_index = element_index for element_index, element in enumerate(detail): element_var = element_vars[element_index] if starred_list_var is not None: statements.insert( starred_index + 1, StatementAssignmentVariable( variable=element_var, source=ExpressionListOperationPop( list_arg=ExpressionTempVariableRef( variable=starred_list_var, source_ref=source_ref), source_ref=source_ref, ), source_ref=source_ref, ), ) elif element[0] != "Starred": statements.append( StatementAssignmentVariable( variable=element_var, source=ExpressionSpecialUnpack( value=ExpressionTempVariableRef( variable=source_iter_var, source_ref=source_ref), count=element_index + 1, expected=starred_index or len(detail), starred=starred_index is not None, source_ref=source_ref, ), source_ref=source_ref, )) else: assert starred_index == element_index starred_list_var = element_var statements.append( StatementAssignmentVariable( variable=element_var, source=ExpressionBuiltinList( value=ExpressionTempVariableRef( variable=source_iter_var, source_ref=source_ref), source_ref=source_ref, ), source_ref=source_ref, )) if starred_list_var is None: statements.append( StatementSpecialUnpackCheck( iterator=ExpressionTempVariableRef( variable=source_iter_var, source_ref=source_ref), count=len(detail), source_ref=source_ref, )) else: statements.insert( starred_index + 1, makeStatementConditional( condition=makeComparisonExpression( comparator="Lt", left=ExpressionBuiltinLen( value=ExpressionTempVariableRef( variable=starred_list_var, source_ref=source_ref), source_ref=source_ref, ), right=makeConstantRefNode( constant=len(statements) - starred_index - 1, source_ref=source_ref, ), source_ref=source_ref, ), yes_branch=makeRaiseExceptionExpressionFromTemplate( exception_type="ValueError", template="""\ not enough values to unpack (expected at least %d, got %%d)""" % (len(statements) - 1), template_args=makeBinaryOperationNode( operator="Add", left=ExpressionBuiltinLen( value=ExpressionTempVariableRef( variable=starred_list_var, source_ref=source_ref), source_ref=source_ref, ), right=makeConstantRefNode(constant=starred_index, source_ref=source_ref), source_ref=source_ref, ), source_ref=source_ref, ).asStatement(), no_branch=None, source_ref=source_ref, ), ) if python_version >= 0x370: iter_creation_class = ExpressionBuiltinIterForUnpack else: iter_creation_class = ExpressionBuiltinIter1 statements = [ StatementAssignmentVariable( variable=source_iter_var, source=iter_creation_class(value=source, source_ref=source_ref), source_ref=source_ref, ), makeTryFinallyStatement( provider=provider, tried=statements, final=(StatementReleaseVariable(variable=source_iter_var, source_ref=source_ref), ), source_ref=source_ref, ), ] # When all is done, copy over to the actual assignment targets, starred # or not makes no difference here anymore. for element_index, element in enumerate(detail): if element[0] == "Starred": element = element[1] element_var = element_vars[element_index] statements.append( buildAssignmentStatementsFromDecoded( provider=provider, kind=element[0], detail=element[1], source=ExpressionTempVariableRef(variable=element_var, source_ref=source_ref), source_ref=source_ref, )) # Need to release temporary variables right after successful # usage. statements.append( StatementDelVariable(variable=element_var, tolerant=True, source_ref=source_ref)) final_statements = [] for element_var in element_vars: final_statements.append( StatementReleaseVariable(variable=element_var, source_ref=source_ref)) return makeTryFinallyStatement( provider=provider, tried=statements, final=final_statements, source_ref=source_ref, ) elif kind == "Starred": raiseSyntaxError( "starred assignment target must be in a list or tuple", source_ref.atColumnNumber(0), ) else: assert False, (kind, source_ref, detail)
def _detectImports(command, user_provided, technical): # This is pretty complicated stuff, with variants to deal with. # pylint: disable=too-many-branches,too-many-locals,too-many-statements # Print statements for stuff to show, the modules loaded. if python_version >= 0x300: command += """ print("\\n".join(sorted( "import %s # sourcefile %s" % (module.__name__, module.__file__) for module in sys.modules.values() if getattr(module, "__file__", None) not in (None, "<frozen>" ))), file = sys.stderr)""" reduced_path = [ path_element for path_element in sys.path if not areSamePaths(path_element, ".") if not areSamePaths( path_element, os.path.dirname(sys.modules["__main__"].__file__)) ] # Make sure the right import path (the one Nuitka binary is running with) # is used. command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" % repr(reduced_path)) + command if str is not bytes: command = command.encode("utf8") _stdout, stderr, exit_code = executeProcess( command=( sys.executable, "-s", "-S", "-v", "-c", "import sys;exec(sys.stdin.read())", ), stdin=command, env=dict(os.environ, PYTHONIOENCODING="utf-8"), ) assert type(stderr) is bytes # Don't let errors here go unnoticed. if exit_code != 0: # An error by the user pressing CTRL-C should not lead to the below output. if b"KeyboardInterrupt" in stderr: general.sysexit("Pressed CTRL-C while detecting early imports.") general.warning( "There is a problem with detecting imports, CPython said:") for line in stderr.split(b"\n"): printError(line) general.sysexit("Error, please report the issue with above output.") result = [] detections = [] for line in stderr.replace(b"\r", b"").split(b"\n"): if line.startswith(b"import "): parts = line.split(b" # ", 2) module_name = parts[0].split(b" ", 2)[1] origin = parts[1].split()[0] if python_version >= 0x300: module_name = module_name.decode("utf8") module_name = ModuleName(module_name) if origin == b"precompiled": # This is a ".pyc" file that was imported, even before we have a # chance to do anything, we need to preserve it. filename = parts[1][len(b"precompiled from "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 3, "precompiled", filename)) elif origin == b"from" and python_version < 0x300: filename = parts[1][len(b"from "):] if str is not bytes: # For consistency, and maybe later reuse filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) else: assert False elif origin == b"sourcefile": filename = parts[1][len(b"sourcefile "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue if filename.endswith(".py"): detections.append((module_name, 2, "sourcefile", filename)) elif filename.endswith(".pyc"): detections.append( (module_name, 3, "precompiled", filename)) elif not filename.endswith("<frozen>"): # Python3 started lying in "__name__" for the "_decimal" # calls itself "decimal", which then is wrong and also # clashes with "decimal" proper if python_version >= 0x300 and module_name == "decimal": module_name = ModuleName("_decimal") detections.append((module_name, 2, "extension", filename)) elif origin == b"dynamically": # Shared library in early load, happens on RPM based systems and # or self compiled Python installations. filename = parts[1][len(b"dynamically loaded from "):] if python_version >= 0x300: filename = filename.decode("utf8") # Do not leave standard library when freezing. if not isStandardLibraryPath(filename): continue detections.append((module_name, 1, "extension", filename)) for module_name, _priority, kind, filename in sorted(detections): if isStandardLibraryNoAutoInclusionModule(module_name): continue if kind == "extension": _detectedExtensionModule( filename=filename, module_name=module_name, result=result, technical=technical, ) elif kind == "precompiled": _detectedPrecompiledFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) elif kind == "sourcefile": _detectedSourceFile( filename=filename, module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) else: assert False, kind return result