def copyFile(source_path, dest_path):
"""Improved version of shutil.copy
This handles errors with a chance to correct them, e.g. on Windows, files might be
locked by running program or virus checkers.
"""
while 1:
try:
shutil.copyfile(source_path, dest_path)
except PermissionError as e:
if e.errno != errno.EACCES:
raise
general.warning("Problem copying file %s:" % e)
try:
reply = raw_input("Retry? (YES/no) ") or "yes"
except EOFError:
reply = "no"
if reply.upper() == "YES":
continue
raise
break
def addDLLInfo(count, source_dir, original_filename, binary_filename,
package_name):
used_dlls = _detectBinaryDLLs(
is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name,
use_cache=use_cache,
update_cache=update_cache,
)
# Allow plugins to prevent inclusion, this may discard things from used_dlls.
Plugins.removeDllDependencies(dll_filename=binary_filename,
dll_filenames=used_dlls)
for dll_filename in sorted(tuple(used_dlls)):
if not os.path.isfile(dll_filename):
if _not_found_dlls:
general.warning("""\
Dependency '%s' could not be found, expect runtime issues. If this is
working with Python, report a Nuitka bug.""" % dll_filename)
_not_found_dlls.add(dll_filename)
used_dlls.remove(dll_filename)
reportProgressBar(binary_filename)
return binary_filename, package_name, used_dlls
def getFunctionCode(
context,
function_identifier,
parameters,
closure_variables,
user_variables,
outline_variables,
temp_variables,
function_doc,
file_scope,
needs_exception_exit,
):
try:
return _getFunctionCode(
context=context,
function_identifier=function_identifier,
parameters=parameters,
closure_variables=closure_variables,
user_variables=user_variables,
outline_variables=outline_variables,
temp_variables=temp_variables,
function_doc=function_doc,
file_scope=file_scope,
needs_exception_exit=needs_exception_exit,
)
except Exception:
general.warning("Problem creating function code %r." %
function_identifier)
raise
def addDLLInfo(count, source_dir, original_filename, binary_filename,
package_name):
used_dlls = detectBinaryDLLs(
is_main_executable=count == 0,
source_dir=source_dir,
original_filename=original_filename,
binary_filename=binary_filename,
package_name=package_name,
use_cache=use_cache,
update_cache=update_cache,
)
# Allow plugins to prevent inclusion, this may discard things from used_dlls.
Plugins.removeDllDependencies(dll_filename=binary_filename,
dll_filenames=used_dlls)
for dll_filename in sorted(tuple(used_dlls)):
if not os.path.isfile(dll_filename):
if _unfound_dlls:
general.warning(
"Dependency '%s' could not be found, you might need to copy it manually."
% dll_filename)
_unfound_dlls.add(dll_filename)
used_dlls.remove(dll_filename)
return binary_filename, used_dlls
def packDistFolderToOnefile(dist_dir, binary_filename):
"""Pack distribution to onefile, i.e. a single file that is directly executable."""
if getOS() == "Linux":
packDistFolderToOnefileLinux(dist_dir, binary_filename)
elif getOS() == "Windows":
packDistFolderToOnefileWindows(dist_dir)
else:
general.warning("Onefile mode is not yet available on '%s'." % getOS())
def _pickCompressor():
if Options.isExperimental("zstd"):
try:
from zstd import ZSTD_compress # pylint: disable=I0021,import-error
except ImportError:
general.warning(
"Onefile mode cannot compress without 'zstd' module on '%s'." % getOS()
)
else:
return b"Y", lambda data: ZSTD_compress(data, 9, getJobLimit())
else:
return b"X", lambda data: data
def packDistFolderToOnefileWindows(dist_dir):
general.warning("Onefile mode is experimental on '%s'." % getOS())
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
onefile_output_filename = getResultFullpath(onefile=True)
# First need to create the bootstrap binary for unpacking.
_runOnefileScons(quiet=not Options.isShowScons())
# Make sure to copy the resources from the created binary to the bootstrap binary, these
# are icons and version information.
copyResourcesFromFileToFile(
source_filename=getResultFullpath(onefile=False),
target_filename=onefile_output_filename,
resource_kinds=(RT_ICON, RT_GROUP_ICON, RT_VERSION),
)
# Now need to append to payload it, potentially compressing it.
compression_indicator, compressor = _pickCompressor()
with open(onefile_output_filename, "ab") as output_file:
# Seeking to end of file seems necessary on Python2 at least, maybe it's
# just that tell reports wrong value initially.
output_file.seek(0, 2)
start_pos = output_file.tell()
output_file.write(b"KA" + compression_indicator)
# Move the binary to start immediately to the start position
start_binary = getResultFullpath(onefile=False)
file_list = getFileList(dist_dir)
file_list.remove(start_binary)
file_list.insert(0, start_binary)
for filename_full in file_list:
filename_relative = os.path.relpath(filename_full, dist_dir)
filename_encoded = filename_relative.encode("utf-16le") + b"\0\0"
output_file.write(filename_encoded)
with open(filename_full, "rb") as input_file:
compressed = compressor(input_file.read())
output_file.write(struct.pack("Q", len(compressed)))
output_file.write(compressed)
# Using empty filename as a terminator.
output_file.write(b"\0\0")
output_file.write(struct.pack("Q", start_pos))
def getLinuxDistribution():
"""Name of the Linux distribution.
We should usually avoid this, and rather test for the feature,
but in some cases it's hard to manage that.
"""
if getOS() != "Linux":
return None, None, None
# singleton, pylint: disable=global-statement
global _linux_distribution_info
if _linux_distribution_info is None:
result = None
base = None
version = None
if os.path.exists("/etc/os-release"):
result, base, version = _parseOsReleaseFileContents(
"/etc/os-release")
elif os.path.exists("/etc/SuSE-release"):
result, base, version = _parseOsReleaseFileContents(
"/etc/SuSE-release")
elif os.path.exists("/etc/issue"):
result, base, version = _parseOsReleaseFileContents("/etc/issue")
if result is None:
from .Execution import check_output
try:
result = check_output(["lsb_release", "-i", "-s"], shell=False)
if str is not bytes:
result = result.decode("utf8")
except OSError:
pass
if result is None:
from nuitka.Tracing import general
general.warning(
"Cannot detect Linux distribution, this may prevent optimization."
)
result = "Unknown"
# Change e.g. "11 (Bullseye)"" to "11".
if version is not None and version.strip():
version = version.split()[0]
_linux_distribution_info = result.title(), base, version
return _linux_distribution_info
def reportProgressBar(item, total=None, update=True):
if Tracing.progress is not None:
try:
if total is not None:
Tracing.progress.updateTotal(total)
Tracing.progress.setCurrent(item)
if update:
Tracing.progress.update()
except Exception: # Catch all the things, pylint: disable=broad-except
# We disable the progress bar now, because it's causing issues.
general.warning("Progress bar disabled due to bug")
closeProgressBar()
def _cleanupClangFormat(filename):
"""Call clang-format on a given filename to format C code.
Args:
filename: What file to re-format.
"""
# Using global here, as this is really a singleton, in
# the form of a module, pylint: disable=global-statement
global warned_clang_format
clang_format_path = (
getExecutablePath("clang-format-10")
or getExecutablePath("clang-format-9")
or getExecutablePath("clang-format-8")
or getExecutablePath("clang-format-7")
)
# Extra ball on Windows, check default installations paths in MSVC and LLVM too.
if not clang_format_path and getOS() == "Windows":
with withEnvironmentPathAdded(
"PATH",
r"C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Tools\Llvm\bin",
r"C:\Program Files\LLVM\bin",
):
clang_format_path = getExecutablePath("clang-format")
if clang_format_path:
subprocess.call(
[
clang_format_path,
"-i",
"-style={BasedOnStyle: llvm, IndentWidth: 4, ColumnLimit: 120}",
filename,
]
)
else:
if not warned_clang_format:
general.warning("Need to install LLVM for C files format.")
warned_clang_format = True
def attemptToMarshal(constant_identifier, constant_value, emit):
""" Try and marshal a value, if so decided. Indicate with return value.
See above for why marshal is only used in problematic cases.
"""
if not isMarshalConstant(constant_value):
return False
marshal_value = marshal.dumps(constant_value)
restored = marshal.loads(marshal_value)
# TODO: The check in isMarshalConstant is currently preventing this from
# happening.
if not compareConstants(constant_value, restored):
general.warning("Problem with marshal of constant %r", constant_value)
return False
emit("%s = PyMarshal_ReadObjectFromString((char *)%s);" %
(constant_identifier, stream_data.getStreamDataCode(marshal_value)))
return True
def makeSourceDirectory(main_module):
""" Get the full list of modules imported, create code for all of them.
"""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches,too-many-locals,too-many-statements
assert main_module.isCompiledPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneUserModules():
if module.getFullName() == any_case_module:
break
else:
general.warning("Not recursing to unused '%s'." % any_case_module)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=ModuleRegistry.getDoneModules())
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
try:
prepared_modules[
c_filename] = CodeGeneration.prepareModuleCode(
global_context=global_context,
module=module,
module_name=module.getFullName(),
)
except Exception:
general.warning("Problem creating code for module %r." %
module)
raise
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[c_filename][1].getConstantCode(0)
# Second pass, generate the actual module code into the files.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
template_values, module_context = prepared_modules[c_filename]
source_code = CodeGeneration.generateModuleCode(
module_context=module_context, template_values=template_values)
writeSourceCode(filename=c_filename, source_code=source_code)
if Options.isShowInclusion():
general.info("Included compiled module '%s'." %
module.getFullName())
elif module.isPythonShlibModule():
target_filename = os.path.join(
OutputDirectories.getStandaloneDirectoryPath(),
*module.getFullName().split("."))
target_filename += Utils.getSharedLibrarySuffix()
target_dir = os.path.dirname(target_filename)
if not os.path.isdir(target_dir):
makePath(target_dir)
shutil.copyfile(module.getFilename(), target_filename)
standalone_entry_points.append((
module.getFilename(),
target_filename,
module.getFullName().getPackageName(),
))
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
general.info("Included uncompiled module '%s'." %
module.getFullName())
else:
assert False, module
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=ConstantCodes.getConstantsDefinitionCode(
context=global_context),
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
ModuleRegistry.getDoneUserModules())
writeSourceCode(filename=os.path.join(source_dir, "__helpers.h"),
source_code=helper_decl_code)
writeSourceCode(filename=os.path.join(source_dir, "__helpers.c"),
source_code=helper_impl_code)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(filename=os.path.join(target_dir, filename),
source_code=source_code)
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 0x300:
command = command.encode("utf8")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args=[sys.executable, "-s", "-S", "-v", tmp_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dict(os.environ, PYTHONIOENCODING="utf_8"),
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf-8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300:
if module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "shlib", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "shlib", filename))
for module_name, _prio, kind, filename in sorted(detections):
if kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "shlib":
_detectedShlibFile(filename=filename, module_name=module_name)
else:
assert False, kind
return result
def packDistFolderToOnefileLinux(dist_dir, binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
with open(apprun_filename, "w") as output_file:
output_file.write("""\
#!/bin/sh
exec $APPDIR/%s $@""" % os.path.basename(binary_filename))
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
if not icon_paths:
if os.path.exists("/usr/share/pixmaps/python.xpm"):
icon_paths.append("/usr/share/pixmaps/python.xpm")
if icon_paths:
extension = os.path.splitext(icon_paths[0])[1].lower()
shutil.copyfile(icon_paths[0], getResultBasepath() + extension)
else:
general.warning(
"Cannot apply onefile unless icon file is specified. Yes, crazy.")
return
with open(getResultBasepath() + ".desktop", "w") as output_file:
output_file.write(
"""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;""" % {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
})
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
onefile_output_filename = getResultFullpath(onefile=True)
# Starting the process while locked, so file handles are not duplicated.
appimagetool_process = subprocess.Popen(
(
getAppImageToolPath(),
dist_dir,
"--comp",
"xz",
"-n",
onefile_output_filename,
),
shell=False,
stderr=getNullOutput(),
stdout=getNullOutput(),
)
# TODO: Exit code should be checked.
result = appimagetool_process.wait()
if not os.path.exists(onefile_output_filename):
sys.exit("Error, expected output file %s not created by AppImage." %
onefile_output_filename)
postprocessing_logger.info("Completed onefile execution.")
assert result == 0, result
def _getSystemStaticLibPythonPath():
# Return driven function with many cases, pylint: disable=too-many-branches,too-many-return-statements
sys_prefix = getSystemPrefixPath()
python_abi_version = python_version_str + getPythonABI()
if isNuitkaPython():
# Nuitka Python has this.
if isWin32Windows():
return os.path.join(
sys_prefix,
"libs",
"python" + python_abi_version.replace(".", "") + ".lib",
)
else:
return os.path.join(
sys_prefix,
"lib",
"libpython" + python_abi_version + ".a",
)
if isWin32Windows():
candidates = [
# Anaconda has this.
os.path.join(
sys_prefix,
"libs",
"libpython" + python_abi_version.replace(".", "") + ".dll.a",
),
# MSYS2 mingw64 Python has this.
os.path.join(
sys_prefix,
"lib",
"libpython" + python_abi_version + ".dll.a",
),
]
for candidate in candidates:
if os.path.exists(candidate):
return candidate
else:
candidate = os.path.join(
sys_prefix, "lib", "libpython" + python_abi_version + ".a"
)
if os.path.exists(candidate):
return candidate
# For Python2 this works. TODO: Figure out Debian and Python3.
if (
python_version < 0x300
and isDebianPackagePython()
and isDebianSuitableForStaticLinking()
):
candidate = locateStaticLinkLibrary("python" + python_abi_version)
else:
candidate = None
if candidate is not None and os.path.exists(candidate):
# Also check libz, can be missing
if not locateStaticLinkLibrary("z"):
general.warning(
"Error, missing libz-dev installation needed for static lib-python."
)
return candidate
# This is not necessarily only for Python3 on Debian, but maybe others as well,
# but that's what's been tested.
if python_version >= 0x300 and isDebianPackagePython() and isDebianBasedLinux():
try:
import sysconfig
candidate = os.path.join(
sysconfig.get_config_var("LIBPL"),
"libpython" + python_abi_version + "-pic.a",
)
if os.path.exists(candidate):
return candidate
except ImportError:
# Cannot detect this properly for Python 2.6, but we don't care much
# about that anyway.
pass
return None
def makeSourceDirectory():
"""Get the full list of modules imported, create code for all of them."""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (
uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()
):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the asked modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneModules():
if module.getFullName() == any_case_module:
break
else:
general.warning(
"Did not follow import to unused '%s', consider include options."
% any_case_module
)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Do some reporting and determine compiled module to work on
compiled_modules = []
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
compiled_modules.append(module)
if Options.isShowInclusion():
inclusion_logger.info(
"Included compiled module '%s'." % module.getFullName()
)
elif module.isPythonExtensionModule():
addExtensionModuleEntryPoint(module)
if Options.isShowInclusion():
inclusion_logger.info(
"Included extension module '%s'." % module.getFullName()
)
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
inclusion_logger.info(
"Included uncompiled module '%s'." % module.getFullName()
)
else:
assert False, module
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=compiled_modules
)
setupProgressBar(
stage="C Source Generation",
unit="module",
total=len(compiled_modules),
)
# Generate code for compiled modules, this can be slow, so do it separately
# with a progress bar.
for module in compiled_modules:
c_filename = module_filenames[module]
reportProgressBar(
item=module.getFullName(),
)
source_code = CodeGeneration.generateModuleCode(
module=module,
data_filename=os.path.basename(c_filename + "onst"), # Really .const
)
writeSourceCode(filename=c_filename, source_code=source_code)
closeProgressBar()
(
helper_decl_code,
helper_impl_code,
constants_header_code,
constants_body_code,
) = CodeGeneration.generateHelpersCode()
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.h"),
source_code=constants_header_code,
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=constants_body_code,
)
def makeSourceDirectory(main_module):
"""Get the full list of modules imported, create code for all of them."""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches
assert main_module.isCompiledPythonModule()
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (
uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()
):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneModules():
if module.getFullName() == any_case_module:
break
else:
general.warning("Not recursing to unused '%s'." % any_case_module)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=ModuleRegistry.getDoneModules()
)
# Generate code for modules.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
source_code = CodeGeneration.generateModuleCode(
module=module,
data_filename=os.path.basename(c_filename + "onst"), # Really .const
)
writeSourceCode(filename=c_filename, source_code=source_code)
if Options.isShowInclusion():
inclusion_logger.info(
"Included compiled module '%s'." % module.getFullName()
)
elif module.isPythonShlibModule():
addShlibEntryPoint(module)
if Options.isShowInclusion():
inclusion_logger.info(
"Included extension module '%s'." % module.getFullName()
)
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
inclusion_logger.info(
"Included uncompiled module '%s'." % module.getFullName()
)
else:
assert False, module
(
helper_decl_code,
helper_impl_code,
constants_header_code,
constants_body_code,
) = CodeGeneration.generateHelpersCode()
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.h"),
source_code=constants_header_code,
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=constants_body_code,
)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(
filename=os.path.join(target_dir, filename), source_code=source_code
)
def namifyConstant(constant):
# Many branches, statements and every case has a return, this is a huge case
# statement, that encodes the naming policy of constants, with often complex
# conditions, pylint: disable=too-many-branches,too-many-return-statements,too-many-statements
if type(constant) is int:
if constant == 0:
result = "int_0"
elif constant > 0:
result = "int_pos_%d" % constant
else:
result = "int_neg_%d" % abs(constant)
if len(result) > 32:
result = _digest(result)
return result
elif type(constant) is long:
if constant == 0:
result = "long_0"
elif constant > 0:
result = "long_pos_%d" % constant
else:
result = "long_neg_%d" % abs(constant)
if len(result) > 32:
result = _digest(result)
return result
elif constant is None:
return "none"
elif constant is True:
return "true"
elif constant is False:
return "false"
elif constant is Ellipsis:
return "ellipsis"
elif type(constant) is str:
return "str_" + _namifyString(constant)
elif type(constant) is bytes:
return "bytes_" + _namifyString(constant)
elif type(constant) is unicode:
if _isAscii(constant):
return "unicode_" + _namifyString(str(constant))
else:
# Others are better digested to not cause compiler trouble
return "unicode_digest_" + _digest(repr(constant))
elif type(constant) is float:
if math.isnan(constant):
return "float_%s_nan" % ("minus" if math.copysign(1, constant) < 0
else "plus")
return "float_%s" % repr(constant).replace(".", "_").replace(
"-", "minus_").replace("+", "")
elif type(constant) is complex:
value = "%s__%s" % (constant.real, constant.imag)
value = value.replace("+", "p").replace("-", "m").replace(".", "_")
if value.startswith("(") and value.endswith(")"):
value = value[1:-1]
return "complex_%s" % value
elif type(constant) is dict:
if constant == {}:
return "dict_empty"
else:
return "dict_" + _digest(repr(constant))
elif type(constant) is set:
if constant == set():
return "set_empty"
else:
return "set_" + _digest(repr(constant))
elif type(constant) is frozenset:
if constant == frozenset():
return "frozenset_empty"
else:
return "frozenset_" + _digest(repr(constant))
elif type(constant) is tuple:
if constant == ():
return "tuple_empty"
else:
try:
result = "_".join(namifyConstant(value) for value in constant)
if len(result) > 60:
result = _digest(repr(constant))
return "tuple_" + result + "_tuple"
except ExceptionCannotNamify:
general.warning("Couldn't namify '%r'" % (constant, ))
return "tuple_" + _digest(repr(constant))
elif type(constant) is list:
if constant == []:
return "list_empty"
else:
try:
result = "_".join(namifyConstant(value) for value in constant)
if len(result) > 60:
result = _digest(repr(constant))
return "list_" + result + "_list"
except ExceptionCannotNamify:
general.warning("Couldn't namify '%r'" % value)
return "list_" + _digest(repr(constant))
elif type(constant) is bytearray:
return "bytearray_" + _digest(repr(constant))
elif type(constant) is xrange:
return "xrange_%s" % (
str(constant)[7 if str is bytes else 6:-1].replace(
" ", "").replace(",", "_").replace("-", "neg"))
elif type(constant) is slice:
return "slice_%s_%s_%s" % (
namifyConstant(constant.start),
namifyConstant(constant.stop),
namifyConstant(constant.step),
)
elif constant in builtin_anon_values:
return "anon_%s" % builtin_anon_values[constant]
elif type(constant) is type:
return "type_%s" % constant.__name__
elif type(constant) is BuiltinFunctionType:
assert constant in builtin_named_values_list
return "builtin_%s" % constant.__name__
elif constant is NotImplemented:
return "type_notimplemented"
else:
raise ExceptionCannotNamify("%r" % constant, type(constant))
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
if str is not bytes:
command = command.encode("utf8")
_stdout, stderr, exit_code = executeProcess(
command=(
sys.executable,
"-s",
"-S",
"-v",
"-c",
"import sys;exec(sys.stdin.read())",
),
stdin=command,
env=dict(os.environ, PYTHONIOENCODING="utf-8"),
)
assert type(stderr) is bytes
# Don't let errors here go unnoticed.
if exit_code != 0:
# An error by the user pressing CTRL-C should not lead to the below output.
if b"KeyboardInterrupt" in stderr:
general.sysexit("Pressed CTRL-C while detecting early imports.")
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"from" and python_version < 0x300:
filename = parts[1][len(b"from "):]
if str is not bytes: # For consistency, and maybe later reuse
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
else:
assert False
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300 and module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "extension", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "extension", filename))
for module_name, _priority, kind, filename in sorted(detections):
if isStandardLibraryNoAutoInclusionModule(module_name):
continue
if kind == "extension":
_detectedExtensionModule(
filename=filename,
module_name=module_name,
result=result,
technical=technical,
)
elif kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
else:
assert False, kind
return result
