def _runOnefileScons(quiet):
source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True)
SconsInterface.cleanSconsDirectory(source_dir)
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=True),
"result_exe": OutputDirectories.getResultFullpath(onefile=True),
"source_dir": source_dir,
"debug_mode": asBoolStr(Options.is_debug),
"unstripped_mode": asBoolStr(Options.isUnstripped()),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"target_arch": getArchitecture(),
"python_prefix": sys.prefix,
"nuitka_src": SconsInterface.getSconsDataPath(),
"compiled_exe": OutputDirectories.getResultFullpath(onefile=False),
}
SconsInterface.setCommonOptions(options)
onefile_env_values = {}
if Options.isWindowsOnefileTempDirMode() or getOS() != "Windows":
onefile_env_values["ONEFILE_TEMP_SPEC"] = Options.getWindowsOnefileTempDirSpec(
use_default=True
)
else:
# Merge version information if possible, to avoid collisions, or deep nesting
# in file system.
product_version = version_resources["ProductVersion"]
file_version = version_resources["FileVersion"]
if product_version != file_version:
effective_version = "%s-%s" % (product_version, file_version)
else:
effective_version = file_version
onefile_env_values["ONEFILE_COMPANY"] = version_resources["CompanyName"]
onefile_env_values["ONEFILE_PRODUCT"] = version_resources["ProductName"]
onefile_env_values["ONEFILE_VERSION"] = effective_version
with withEnvironmentVarsOverriden(onefile_env_values):
result = SconsInterface.runScons(
options=options, quiet=quiet, scons_filename="WindowsOnefile.scons"
)
# Exit if compilation failed.
if not result:
scons_logger.sysexit("Error, one file bootstrap build for Windows failed.")
if Options.isRemoveBuildDir():
general.info("Removing onefile build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping onefile build directory %r." % source_dir)
def _runCPgoBinary():
# Note: For exit codes, we do not insist on anything yet, maybe we could point it out
# or ask people to make scripts that buffer these kinds of errors, and take an error
# instead as a serious failure.
general.info("Running created binary to produce C level PGO information:",
style="blue")
if _wasMsvcMode():
msvc_pgc_filename = _deleteMsvcPGOFiles(pgo_mode="generate")
with withEnvironmentVarOverridden(
"PATH",
getSconsReportValue(
source_dir=OutputDirectories.getSourceDirectoryPath(),
key="PATH"),
):
_exit_code = _runPgoBinary()
pgo_data_collected = os.path.exists(msvc_pgc_filename)
else:
_exit_code = _runPgoBinary()
gcc_constants_pgo_filename = os.path.join(
OutputDirectories.getSourceDirectoryPath(), "__constants.gcda")
pgo_data_collected = os.path.exists(gcc_constants_pgo_filename)
if not pgo_data_collected:
general.sysexit(
"Error, no PGO information produced, did the created binary run at all?"
)
general.info("Successfully collected C level PGO information.",
style="blue")
def generateBytecodeFrozenCode():
frozen_defs = []
for uncompiled_module in getUncompiledTechnicalModules():
module_name = uncompiled_module.getFullName()
code_data = uncompiled_module.getByteCode()
is_package = uncompiled_module.isUncompiledPythonPackage()
size = len(code_data)
# Packages are indicated with negative size.
if is_package:
size = -size
frozen_defs.append("""\
{{"{module_name}", {start}, {size}}},""".format(
module_name=module_name,
start=stream_data.getStreamDataOffset(code_data),
size=size,
))
if Options.isShowInclusion():
general.info("Embedded as frozen module '%s'." % module_name)
if frozen_defs:
return template_frozen_modules % {
"frozen_modules": indented(frozen_defs, 2)
}
else:
return None
def optimizeCompiledPythonModule(module):
if _progress:
progress_logger.info(
"Doing module local optimizations for '{module_name}'.".format(
module_name=module.getFullName()))
touched = False
if _progress and Options.isShowMemory():
memory_watch = MemoryUsage.MemoryWatch()
while True:
tag_set.clear()
try:
# print("Compute module")
module.computeModule()
except BaseException:
general.info("Interrupted while working on '%s'." % module)
raise
Graphs.onModuleOptimizationStep(module)
# Search for local change tags.
for tag in tag_set:
if tag == "new_code":
continue
break
else:
if _progress:
progress_logger.info("Finished with the module.")
break
if _progress:
if "new_code" in tag_set:
tag_set.remove("new_code")
progress_logger.info(
"Not finished with the module due to following change kinds: %s"
% ",".join(sorted(tag_set)))
# Otherwise we did stuff, so note that for return value.
touched = True
if _progress and Options.isShowMemory():
memory_watch.finish()
memory_logger.info(
"Memory usage changed during optimization of '%s': %s" %
(module.getFullName(), memory_watch.asStr()))
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
return touched
def compileTree(main_module):
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before generating C code: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
# Now build the target language code for the whole tree.
makeSourceDirectory(main_module=main_module)
bytecode_accessor = ConstantAccessor(
data_filename="__bytecode.const", top_level_name="bytecode_data"
)
# This should take all bytecode values, even ones needed for frozen or
# not produce anything.
loader_code = LoaderCodes.getMetapathLoaderBodyCode(bytecode_accessor)
writeSourceCode(
filename=os.path.join(source_dir, "__loader.c"), source_code=loader_code
)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
sys.exit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.isDebug():
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
# TODO: On Windows, we could run this in parallel to Scons, on Linux we need it
# for linking.
runDataComposer(source_dir)
# Run the Scons to build things.
result, options = runScons(main_module=main_module, quiet=not Options.isShowScons())
return result, options
def compileTree(main_module):
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before generating C code: {memory}:".
format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()))
# Now build the target language code for the whole tree.
makeSourceDirectory(main_module=main_module)
frozen_code = generateBytecodeFrozenCode()
if frozen_code is not None:
writeSourceCode(filename=os.path.join(source_dir, "__frozen.c"),
source_code=frozen_code)
if not isWin32Windows():
writeBinaryData(
filename=os.path.join(source_dir, "__constants.bin"),
binary_data=ConstantCodes.stream_data.getBytes(),
)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
sys.exit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()))
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.isDebug():
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
# Run the Scons to build things.
result, options = runScons(main_module=main_module,
quiet=not Options.isShowScons())
return result, options
def packDistFolderToOnefileBootstrap(onefile_output_filename, dist_dir):
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while."
)
general.info("Running bootstrap binary compilation via Scons.")
# First need to create the bootstrap binary for unpacking.
_runOnefileScons(quiet=not Options.isShowScons())
executePostProcessingResources(manifest=None, onefile=True)
# Now need to append to payload it, potentially compressing it.
compression_indicator, compressor = _pickCompressor()
with open(onefile_output_filename, "ab") as output_file:
# Seeking to end of file seems necessary on Python2 at least, maybe it's
# just that tell reports wrong value initially.
output_file.seek(0, 2)
start_pos = output_file.tell()
output_file.write(b"KA" + compression_indicator)
# Move the binary to start immediately to the start position
start_binary = getResultFullpath(onefile=False)
file_list = getFileList(dist_dir, normalize=False)
file_list.remove(start_binary)
file_list.insert(0, start_binary)
for filename_full in file_list:
filename_relative = os.path.relpath(filename_full, dist_dir)
filename_encoded = filename_relative.encode("utf-16le") + b"\0\0"
output_file.write(filename_encoded)
with open(filename_full, "rb") as input_file:
compressed = compressor(input_file.read())
output_file.write(struct.pack("Q", len(compressed)))
output_file.write(compressed)
# Using empty filename as a terminator.
output_file.write(b"\0\0")
output_file.write(struct.pack("Q", start_pos))
def detectBinaryPathDLLsWindowsDependencyWalker(
is_main_executable,
source_dir,
original_dir,
binary_filename,
package_name,
use_cache,
update_cache,
):
# This is the caching mechanism and plugin handling for DLL imports.
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="depends.exe",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
result = OrderedSet()
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
if Options.isShowProgress():
general.info("Analysing dependencies of '%s'." % binary_filename)
scan_dirs = getScanDirectories(package_name, original_dir)
result = detectDLLsWithDependencyWalker(binary_filename, scan_dirs)
if update_cache:
putTextFileContents(filename=cache_filename, contents=result)
return result
def writeCompilationReport(report_filename):
active_modules_info = getModuleInclusionInfos()
root = TreeXML.Element("nuitka-compilation-report")
for module in getDoneModules():
active_module_info = active_modules_info[module]
root.append(
TreeXML.Element(
"module",
name=module.getFullName(),
kind=module.__class__.__name__,
reason=active_module_info.reason,
))
putTextFileContents(filename=report_filename,
contents=TreeXML.toString(root))
general.info("Compilation report in file %r." % report_filename)
def writeCompilationReport(report_filename):
active_modules_info = getModuleInclusionInfos()
root = TreeXML.Element("nuitka-compilation-report")
for module in getDoneModules():
active_module_info = active_modules_info[module]
root.append(
TreeXML.Element(
"module",
name=module.getFullName(),
kind=module.__class__.__name__,
reason=active_module_info.reason,
))
for included_datafile in getIncludedDataFiles():
if included_datafile.kind == "data_file":
root.append(
TreeXML.Element(
"data_file",
name=included_datafile.dest_path,
source=included_datafile.source_path,
reason=included_datafile.reason,
tags=",".join(included_datafile.tags),
))
elif included_datafile.kind == "data_blob":
root.append(
TreeXML.Element(
"data_blob",
name=included_datafile.dest_path,
reason=included_datafile.reason,
tags=",".join(included_datafile.tags),
))
putTextFileContents(filename=report_filename,
contents=TreeXML.toString(root))
general.info("Compilation report in file %r." % report_filename)
def main():
"""Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches,too-many-statements
# In case we are in a PGO run, we read its information first, so it becomes
# available for later parts.
pgo_filename = getPythonPgoInput()
if pgo_filename is not None:
readPGOInputFile(pgo_filename)
if not Options.shallDumpBuiltTreeXML():
general.info(
"Starting Python compilation with Nuitka %r on Python %r commercial %r."
% (getNuitkaVersion(), python_version_str, getCommercialVersion())
)
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
addIncludedDataFilesFromFileOptions()
# Turn that source code into a node tree structure.
try:
main_module = _createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
addIncludedDataFilesFromPackageOptions()
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree()
# Exit if compilation failed.
if not result:
sys.exit(1)
# Relaunch in case of Python PGO input to be produced.
if Options.shallCreatePgoInput():
# Will not return.
pgo_filename = OutputDirectories.getPgoRunInputFilename()
general.info(
"Restarting compilation using collected information from '%s'."
% pgo_filename
)
reExecuteNuitka(pgo_filename=pgo_filename)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing()
copyDataFiles()
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
setMainEntryPoint(binary_filename)
dist_dir = OutputDirectories.getStandaloneDirectoryPath()
for module in ModuleRegistry.getDoneModules():
addIncludedEntryPoints(Plugins.considerExtraDlls(dist_dir, module))
copyDllsUsed(
source_dir=OutputDirectories.getSourceDirectoryPath(),
dist_dir=dist_dir,
standalone_entry_points=getStandaloneEntryPoints(),
)
Plugins.onStandaloneDistributionFinished(dist_dir)
if Options.isOnefileMode():
packDistFolderToOnefile(dist_dir, binary_filename)
if Options.isRemoveBuildDir():
general.info("Removing dist folder %r." % dist_dir)
removeDirectory(path=dist_dir, ignore_errors=False)
else:
general.info(
"Keeping dist folder %r for inspection, no need to use it."
% dist_dir
)
# Remove the source directory (now build directory too) if asked to.
source_dir = OutputDirectories.getSourceDirectoryPath()
if Options.isRemoveBuildDir():
general.info("Removing build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping build directory %r." % source_dir)
final_filename = OutputDirectories.getResultFullpath(
onefile=Options.isOnefileMode()
)
if Options.isStandaloneMode() and isMacOS():
general.info(
"Created binary that runs on macOS %s (%s) or higher."
% (options["macos_min_version"], options["macos_target_arch"])
)
Plugins.onFinalResult(final_filename)
general.info("Successfully created %r." % final_filename)
report_filename = Options.getCompilationReportFilename()
if report_filename:
writeCompilationReport(report_filename)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
run_filename = OutputDirectories.getResultRunFilename(
onefile=Options.isOnefileMode()
)
general.info("Launching %r." % run_filename)
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=run_filename,
clean_path=Options.shallClearPythonPathEnvironment(),
)
def compileTree():
source_dir = OutputDirectories.getSourceDirectoryPath()
general.info("Completed Python level compilation and optimization.")
if not Options.shallOnlyExecCCompilerCall():
general.info("Generating source code for C backend compiler.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before generating C code: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
# Now build the target language code for the whole tree.
makeSourceDirectory()
bytecode_accessor = ConstantAccessor(
data_filename="__bytecode.const", top_level_name="bytecode_data"
)
# This should take all bytecode values, even ones needed for frozen or
# not produce anything.
loader_code = LoaderCodes.getMetapathLoaderBodyCode(bytecode_accessor)
writeSourceCode(
filename=os.path.join(source_dir, "__loader.c"), source_code=loader_code
)
else:
source_dir = OutputDirectories.getSourceDirectoryPath()
if not os.path.isfile(os.path.join(source_dir, "__helpers.h")):
general.sysexit("Error, no previous build directory exists.")
if Options.isShowProgress() or Options.isShowMemory():
general.info(
"Total memory usage before running scons: {memory}:".format(
memory=MemoryUsage.getHumanReadableProcessMemoryUsage()
)
)
if Options.isShowMemory():
InstanceCounters.printStats()
if Options.is_debug:
Reports.doMissingOptimizationReport()
if Options.shallNotDoExecCCompilerCall():
return True, {}
general.info("Running data composer tool for optimal constant value handling.")
runDataComposer(source_dir)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(
filename=os.path.join(target_dir, filename), source_code=source_code
)
general.info("Running C compilation via Scons.")
# Run the Scons to build things.
result, options = runSconsBackend(quiet=not Options.isShowScons())
return result, options
def _runOnefileScons(quiet):
# Scons gets transported many details, that we express as variables, and
# have checks for them, leading to many branches and statements,
# pylint: disable=too-many-branches,too-many-statements
source_dir = OutputDirectories.getSourceDirectoryPath(onefile=True)
SconsInterface.cleanSconsDirectory(source_dir)
asBoolStr = SconsInterface.asBoolStr
options = {
"result_name": OutputDirectories.getResultBasepath(onefile=True),
"result_exe": OutputDirectories.getResultFullpath(onefile=True),
"source_dir": source_dir,
"debug_mode": asBoolStr(Options.is_debug),
"unstripped_mode": asBoolStr(Options.isUnstripped()),
"experimental": ",".join(Options.getExperimentalIndications()),
"trace_mode": asBoolStr(Options.shallTraceExecution()),
"target_arch": getArchitecture(),
"python_prefix": sys.prefix,
"nuitka_src": SconsInterface.getSconsDataPath(),
"compiled_exe": OutputDirectories.getResultFullpath(onefile=False),
}
# Ask Scons to cache on Windows, except where the directory is thrown
# away. On non-Windows you can should use ccache instead.
if not Options.isRemoveBuildDir() and getOS() == "Windows":
options["cache_mode"] = "true"
if Options.isLto():
options["lto_mode"] = "true"
if Options.shallDisableConsoleWindow():
options["win_disable_console"] = "true"
if Options.isShowScons():
options["show_scons"] = "true"
if Options.isMingw64():
options["mingw_mode"] = "true"
if Options.getMsvcVersion():
msvc_version = Options.getMsvcVersion()
msvc_version = msvc_version.replace("exp", "Exp")
if "." not in msvc_version:
msvc_version += ".0"
options["msvc_version"] = msvc_version
if getOS() == "Windows":
options["noelf_mode"] = "true"
if Options.isClang():
options["clang_mode"] = "true"
cpp_defines = Plugins.getPreprocessorSymbols()
if cpp_defines:
options["cpp_defines"] = ",".join(
"%s%s%s" % (key, "=" if value else "", value or "")
for key, value in cpp_defines.items())
link_libraries = Plugins.getExtraLinkLibraries()
if link_libraries:
options["link_libraries"] = ",".join(link_libraries)
if Options.shallRunInDebugger():
options["full_names"] = "true"
if Options.assumeYesForDownloads():
options["assume_yes_for_downloads"] = "true"
onefile_env_values = {}
if not Options.isWindowsOnefileTempDirMode():
# Merge version information if necessary, to avoid collisions, or deep nesting
# in file system.
product_version = version_resources["ProductVersion"]
file_version = version_resources["FileVersion"]
if product_version != file_version:
effective_version = "%s-%s" % (product_version, file_version)
else:
effective_version = file_version
onefile_env_values["ONEFILE_COMPANY"] = version_resources[
"CompanyName"]
onefile_env_values["ONEFILE_PRODUCT"] = version_resources[
"ProductName"]
onefile_env_values["ONEFILE_VERSION"] = effective_version
with withEnvironmentVarsOverriden(onefile_env_values):
result = SconsInterface.runScons(options=options,
quiet=quiet,
scons_filename="WindowsOnefile.scons")
# Exit if compilation failed.
if not result:
scons_logger.sysexit(
"Error, one file bootstrap build for Windows failed.")
if Options.isRemoveBuildDir():
general.info("Removing onefile build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping onefile build directory %r." % source_dir)
def optimizeCompiledPythonModule(module):
optimization_logger.info_fileoutput(
"Doing module local optimizations for '{module_name}'.".format(
module_name=module.getFullName()),
other_logger=progress_logger,
)
touched = False
if _progress and Options.isShowMemory():
memory_watch = MemoryWatch()
# Temporary workaround, since we do some optimization based on the last pass results
# that are then not yet fully seen in the traces yet until another time around, we
# allow to continue the loop even without changes one more time.
unchanged_count = 0
while True:
tag_set.clear()
try:
# print("Compute module")
with withChangeIndicationsTo(signalChange):
scopes_were_incomplete = module.computeModule()
except BaseException:
general.info("Interrupted while working on '%s'." % module)
raise
if scopes_were_incomplete:
tag_set.add("var_usage")
Graphs.onModuleOptimizationStep(module)
# Ignore other modules brought into the game.
if "new_code" in tag_set:
tag_set.remove("new_code")
# Search for local change tags.
if not tag_set:
unchanged_count += 1
if unchanged_count == 1 and pass_count == 1:
optimization_logger.info_fileoutput(
"No changed, but retrying one more time.",
other_logger=progress_logger,
)
continue
optimization_logger.info_fileoutput("Finished with the module.",
other_logger=progress_logger)
break
unchanged_count = 0
optimization_logger.info_fileoutput(
"Not finished with the module due to following change kinds: %s" %
",".join(sorted(tag_set)),
other_logger=progress_logger,
)
# Otherwise we did stuff, so note that for return value.
touched = True
if _progress and Options.isShowMemory():
memory_watch.finish()
memory_logger.info(
"Memory usage changed during optimization of '%s': %s" %
(module.getFullName(), memory_watch.asStr()))
Plugins.considerImplicitImports(module=module, signal_change=signalChange)
return touched
def makeSourceDirectory(main_module):
""" Get the full list of modules imported, create code for all of them.
"""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches,too-many-locals,too-many-statements
assert main_module.isCompiledPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneUserModules():
if module.getFullName() == any_case_module:
break
else:
general.warning("Not recursing to unused '%s'." % any_case_module)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=ModuleRegistry.getDoneModules())
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
try:
prepared_modules[
c_filename] = CodeGeneration.prepareModuleCode(
global_context=global_context,
module=module,
module_name=module.getFullName(),
)
except Exception:
general.warning("Problem creating code for module %r." %
module)
raise
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[c_filename][1].getConstantCode(0)
# Second pass, generate the actual module code into the files.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
template_values, module_context = prepared_modules[c_filename]
source_code = CodeGeneration.generateModuleCode(
module_context=module_context, template_values=template_values)
writeSourceCode(filename=c_filename, source_code=source_code)
if Options.isShowInclusion():
general.info("Included compiled module '%s'." %
module.getFullName())
elif module.isPythonShlibModule():
target_filename = os.path.join(
OutputDirectories.getStandaloneDirectoryPath(),
*module.getFullName().split("."))
target_filename += Utils.getSharedLibrarySuffix()
target_dir = os.path.dirname(target_filename)
if not os.path.isdir(target_dir):
makePath(target_dir)
shutil.copyfile(module.getFilename(), target_filename)
standalone_entry_points.append((
module.getFilename(),
target_filename,
module.getFullName().getPackageName(),
))
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
general.info("Included uncompiled module '%s'." %
module.getFullName())
else:
assert False, module
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=ConstantCodes.getConstantsDefinitionCode(
context=global_context),
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
ModuleRegistry.getDoneUserModules())
writeSourceCode(filename=os.path.join(source_dir, "__helpers.h"),
source_code=helper_decl_code)
writeSourceCode(filename=os.path.join(source_dir, "__helpers.c"),
source_code=helper_impl_code)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(filename=os.path.join(target_dir, filename),
source_code=source_code)
def main():
"""Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches,too-many-statements
if not Options.shallDumpBuiltTreeXML():
general.info("Starting Python compilation.")
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
# Turn that source code into a node tree structure.
try:
main_module = _createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree()
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing()
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = OutputDirectories.getResultBasepath() + ".pyi"
putTextFileContents(
filename=pyi_filename,
contents="""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
"""
% {
"imports": "\n".join(
"import %s" % module_name for module_name in getImportedNames()
)
},
)
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
setMainEntryPoint(binary_filename)
dist_dir = OutputDirectories.getStandaloneDirectoryPath()
for module in ModuleRegistry.getDoneModules():
addIncludedEntryPoints(Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(
source_dir=OutputDirectories.getSourceDirectoryPath(),
dist_dir=dist_dir,
standalone_entry_points=getStandaloneEntryPoints(),
)
copyDataFiles(dist_dir=dist_dir)
Plugins.onStandaloneDistributionFinished(dist_dir)
if Options.isOnefileMode():
packDistFolderToOnefile(dist_dir, binary_filename)
if Options.isRemoveBuildDir():
general.info("Removing dist folder %r." % dist_dir)
removeDirectory(path=dist_dir, ignore_errors=False)
else:
general.info(
"Keeping dist folder %r for inspection, no need to use it."
% dist_dir
)
# Remove the source directory (now build directory too) if asked to.
source_dir = OutputDirectories.getSourceDirectoryPath()
if Options.isRemoveBuildDir():
general.info("Removing build directory %r." % source_dir)
removeDirectory(path=source_dir, ignore_errors=False)
assert not os.path.exists(source_dir)
else:
general.info("Keeping build directory %r." % source_dir)
final_filename = OutputDirectories.getResultFullpath(
onefile=Options.isOnefileMode()
)
Plugins.onFinalResult(final_filename)
general.info("Successfully created %r." % final_filename)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
general.info("Launching %r." % final_filename)
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=final_filename,
clean_path=Options.shallClearPythonPathEnvironment(),
)
