def copyDataFiles(dist_dir):
""" Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
for _plugin_name, (source_desc, target_filename) in Plugins.considerDataFiles(
module
):
target_filename = os.path.join(dist_dir, target_filename)
assert isPathBelow(dist_dir, target_filename)
makePath(os.path.dirname(target_filename))
if inspect.isfunction(source_desc):
content = source_desc(target_filename)
if content is not None: # support creation of empty directories
with open(
target_filename, "wb" if type(content) is bytes else "w"
) as output:
output.write(content)
else:
shutil.copy2(source_desc, target_filename)
def addIncludedDataFilesFromPackageOptions():
"""Late data files, from plugins and user options that work with packages"""
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonPackage(
) or module.isUncompiledPythonPackage():
package_name = module.getFullName()
match, reason = package_name.matchesToShellPatterns(
patterns=getShallIncludePackageData())
if match:
package_directory = module.getCompileTimeDirectory()
pkg_filenames = getFileList(
package_directory,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
getSharedLibrarySuffixes(),
)
if pkg_filenames:
file_reason = "package '%s' %s" % (package_name, reason)
for pkg_filename in pkg_filenames:
rel_path = os.path.join(
package_name.asPath(),
os.path.relpath(pkg_filename, package_directory),
)
addIncludedDataFile(
makeIncludedDataFile(
source_path=pkg_filename,
dest_path=rel_path,
reason=file_reason,
tracer=options_logger,
tags="user,package_data",
))
# Circular dependency
from nuitka.plugins.Plugins import Plugins
# Plugins provide per module through this.
for module in ModuleRegistry.getDoneModules():
for included_datafile in Plugins.considerDataFiles(module=module):
addIncludedDataFile(included_datafile)
for included_datafile in getIncludedDataFiles():
if isinstance(included_datafile, (IncludedDataFile)):
Plugins.onDataFileTags(included_datafile, )
def copyDataFiles(dist_dir):
"""Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
# Many details to deal with, pylint: disable=too-many-branches,too-many-locals
for pattern, dest, arg in Options.getShallIncludeDataFiles():
filenames = resolveShellPatternToFilenames(pattern)
if not filenames:
inclusion_logger.warning("No match data file to be included: %r" %
pattern)
for filename in filenames:
file_reason = "specified data file %r on command line" % arg
rel_path = dest
if rel_path.endswith(("/", os.path.sep)):
rel_path = os.path.join(rel_path, os.path.basename(filename))
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
for src, dest in Options.getShallIncludeDataDirs():
filenames = getFileList(src)
if not filenames:
inclusion_logger.warning("No files in directory" % src)
for filename in filenames:
relative_filename = relpath(filename, src)
file_reason = "specified data dir %r on command line" % src
rel_path = os.path.join(dest, relative_filename)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
for plugin, included_datafile in Plugins.considerDataFiles(module):
_handleDataFile(dist_dir=dist_dir,
tracer=plugin,
included_datafile=included_datafile)
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonPackage(
) or module.isUncompiledPythonPackage():
package_name = module.getFullName()
match, reason = package_name.matchesToShellPatterns(
patterns=Options.getShallIncludePackageData())
if match:
package_directory = module.getCompileTimeDirectory()
pkg_filenames = getFileList(
package_directory,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
getSharedLibrarySuffixes(),
)
if pkg_filenames:
file_reason = "package '%s' %s" % (package_name, reason)
for pkg_filename in pkg_filenames:
rel_path = os.path.join(
package_name.asPath(),
os.path.relpath(pkg_filename, package_directory),
)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(pkg_filename, rel_path,
file_reason),
)
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir = os.path.dirname(os.path.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt"
)
if os.path.isfile(origin_prefix_filename):
data_files.append(
(filename, "orig-prefix.txt")
)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(
filename = filename
)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree(
main_module = main_module
)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
standalone_entry_points.insert(
0,
(binary_filename, binary_filename, None)
)
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
copyUsedDLLs(
source_dir = getSourceDirectoryPath(main_module),
dist_dir = dist_dir,
standalone_entry_points = standalone_entry_points
)
for module in ModuleRegistry.getDoneModules():
data_files.extend(
Plugins.considerDataFiles(module)
)
for source_filename, target_filename in data_files:
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module),
target_filename
)
makePath(os.path.dirname(target_filename))
shutil.copy2(
source_filename,
target_filename
)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(
path = getSourceDirectoryPath(main_module),
ignore_errors = False
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here. TODO: Move inside scons file and avoid subprocess call.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(
(
"chmod",
"-x",
getResultFullpath(main_module)
)
)
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = getResultBasepath(main_module) + ".pyi"
with open(pyi_filename, 'w') as pyi_file:
pyi_file.write(
"""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
""" % {
"imports" : '\n'.join(
"import %s" % module_name
for module_name in
getImportedNames()
)
}
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree = main_module,
clean_path = Options.shallClearPythonPathEnvironment()
)
else:
executeMain(
binary_filename = getResultFullpath(main_module),
clean_path = Options.shallClearPythonPathEnvironment()
)
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=Utils.dirname(Utils.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt")
if Utils.isFile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort.
sys.exit(SyntaxErrors.formatOutput(e))
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(getSourceDirectoryPath(main_module))
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(0, (binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = Utils.joinpath(
getStandaloneDirectoryPath(main_module), target_filename)
Utils.makePath(Utils.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment())
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment())
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(main_dir=Utils.dirname(Utils.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()), "orig-prefix.txt"
)
if Utils.isFile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort.
sys.exit(SyntaxErrors.formatOutput(e))
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(getSourceDirectoryPath(main_module))
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(0, (binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(dist_dir=dist_dir, standalone_entry_points=standalone_entry_points)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = Utils.joinpath(getStandaloneDirectoryPath(main_module), target_filename)
Utils.makePath(Utils.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(tree=main_module, clean_path=Options.shallClearPythonPathEnvironment())
else:
executeMain(
binary_filename=getResultFullpath(main_module), clean_path=Options.shallClearPythonPathEnvironment()
)
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
handleSyntaxError(e)
if Options.shallDumpBuiltTreeXML():
# XML output only.
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
else:
# Make the actual compilation.
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCCompilerCall():
if Options.isShowMemory():
MemoryUsage.showMemoryTrace()
sys.exit(0)
executePostProcessing(getResultFullpath(main_module))
if Options.isStandaloneMode():
binary_filename = options["result_exe"]
standalone_entry_points.insert(0, (binary_filename, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
copyUsedDLLs(
source_dir=getSourceDirectoryPath(main_module),
dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points,
)
data_files = []
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
copyDataFiles(dist_dir=dist_dir, data_files=data_files)
Plugins.onStandaloneDistributionFinished(dist_dir)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(
path=getSourceDirectoryPath(main_module), ignore_errors=False
)
if Options.shallMakeModule() and Options.shallCreatePyiFile():
pyi_filename = getResultBasepath(main_module) + ".pyi"
with open(pyi_filename, "w") as pyi_file:
pyi_file.write(
"""\
# This file was generated by Nuitka and describes the types of the
# created shared library.
# At this time it lists only the imports made and can be used by the
# tools that bundle libraries, including Nuitka itself. For instance
# standalone mode usage of the created library will need it.
# In the future, this will also contain type information for values
# in the module, so IDEs will use this. Therefore please include it
# when you make software releases of the extension module that it
# describes.
%(imports)s
# This is not Python source even if it looks so. Make it clear for
# now. This was decided by PEP 484 designers.
__name__ = ...
"""
% {
"imports": "\n".join(
"import %s" % module_name
for module_name in getImportedNames()
)
}
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment(),
)
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment(),
)
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt")
if os.path.isfile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort. If
# we are not in full compat, and user has not specified the Python
# versions he wants, tell him about the potential version problem.
error_message = SyntaxErrors.formatOutput(e)
if not Options.isFullCompat() and \
Options.getIntendedPythonVersion() is None:
if python_version < 300:
suggested_python_version_str = getSupportedPythonVersions()[-1]
else:
suggested_python_version_str = "2.7"
error_message += """
Nuitka is very syntax compatible with standard Python. It is currently running
with Python version '%s', you might want to specify more clearly with the use
of e.g. '--python-version=%s' option, if that's not the one expected.
""" % (python_version_str, suggested_python_version_str)
sys.exit(error_message)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(path=getSourceDirectoryPath(main_module),
ignore_errors=False)
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(0, (None, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module), target_filename)
makePath(os.path.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment())
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment())
