def _detectedPrecompiledFile(filename, module_name, result, user_provided,
technical):
if filename.endswith(".pyc"):
if os.path.isfile(filename[:-1]):
return _detectedSourceFile(
filename=filename[:-1],
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
if module_name in module_names:
return
if Options.isShowInclusion():
inclusion_logger.info("Freezing module '%s' (from '%s')." %
(module_name, filename))
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=loadCodeObjectData(precompiled_filename=filename),
is_package="__init__" in filename,
filename=filename,
user_provided=user_provided,
technical=technical,
)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def _copyDllsUsed(dist_dir, used_dlls):
setupProgressBar(
stage="Copying used DLLs",
unit="DLL",
total=len(used_dlls),
)
dll_map = []
for dll_filename, (package_name, sources) in iterItems(used_dlls):
dll_name = os.path.basename(dll_filename)
target_path = os.path.join(dist_dir, dll_name)
reportProgressBar(target_path)
# Sometimes DLL dependencies were copied there already. TODO: That should
# actually become disallowed with plugins no longer seeing that folder.
if not os.path.exists(target_path):
copyDllFile(source_path=dll_filename, dest_path=target_path)
dll_map.append((dll_filename, package_name, dll_name))
if Options.isShowInclusion():
inclusion_logger.info(
"Included used shared library '%s' (used by %s)." %
(dll_filename, ", ".join(sources)))
closeProgressBar()
return dll_map
def getMetapathLoaderBodyCode(bytecode_accessor):
metapath_loader_inittab = []
metapath_module_decls = []
for other_module in getDoneModules():
metapath_loader_inittab.append(
getModuleMetapathLoaderEntryCode(
module=other_module, bytecode_accessor=bytecode_accessor
)
)
if other_module.isCompiledPythonModule():
metapath_module_decls.append(
"""\
extern PyObject *modulecode_%(module_identifier)s(PyObject *, struct Nuitka_MetaPathBasedLoaderEntry const *);"""
% {"module_identifier": other_module.getCodeName()}
)
for uncompiled_module in getUncompiledModules():
metapath_loader_inittab.append(
getModuleMetapathLoaderEntryCode(
module=uncompiled_module, bytecode_accessor=bytecode_accessor
)
)
frozen_defs = []
for uncompiled_module in getUncompiledTechnicalModules():
module_name = uncompiled_module.getFullName()
code_data = uncompiled_module.getByteCode()
is_package = uncompiled_module.isUncompiledPythonPackage()
size = len(code_data)
# Packages are indicated with negative size.
if is_package:
size = -size
accessor_code = bytecode_accessor.getBlobDataCode(code_data)
frozen_defs.append(
"""\
{{"{module_name}", {start}, {size}}},""".format(
module_name=module_name,
start=accessor_code[accessor_code.find("[") + 1 : -1],
size=size,
)
)
if Options.isShowInclusion():
inclusion_logger.info("Embedded as frozen module '%s'." % module_name)
return template_metapath_loader_body % {
"metapath_module_decls": indented(metapath_module_decls, 0),
"metapath_loader_inittab": indented(metapath_loader_inittab),
"bytecode_count": bytecode_accessor.getConstantsCount(),
"frozen_modules": indented(frozen_defs),
}
def setSharedLibraryRPATH(filename, rpath):
if Options.isShowInclusion():
inclusion_logger.info("Setting 'RPATH' value '%s' for '%s'." %
(rpath, filename))
with withMadeWritableFileMode(filename):
if isMacOS():
_setSharedLibraryRPATHDarwin(filename, rpath)
else:
_setSharedLibraryRPATHElf(filename, rpath)
def removeSharedLibraryRPATH(filename):
rpath = getSharedLibraryRPATH(filename)
if rpath is not None:
if Options.isShowInclusion():
inclusion_logger.info("Removing 'RPATH' setting '%s' from '%s'." %
(rpath, filename))
with withMadeWritableFileMode(filename):
if getOS() == "Darwin":
return _removeSharedLibraryRPATHDarwin(filename, rpath)
else:
return _removeSharedLibraryRPATHElf(filename)
def demoteCompiledModuleToBytecode(module):
"""Demote a compiled module to uncompiled (bytecode)."""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
if Options.isShowProgress():
inclusion_logger.info("Demoting module %r to bytecode from %r." %
(full_name, filename))
source_code = module.getSourceCode()
# Second chance for plugins to modify source code just before turning it
# to bytecode.
source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name,
is_package=False,
source_code=source_code)
bytecode = compile(source_code, filename, "exec", dont_inherit=True)
bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name,
is_package=False,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=full_name,
filename=filename,
bytecode=marshal.dumps(bytecode),
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
uncompiled_module.setUsedModules(module.getUsedModules())
module.finalize()
if isImportedModuleByName(full_name):
replaceImportedModule(old=module, new=uncompiled_module)
replaceRootModule(old=module, new=uncompiled_module)
from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule
if isTriggerModule(module):
replaceTriggerModule(old=module, new=uncompiled_module)
def demoteCompiledModuleToBytecode(module):
"""Demote a compiled module to uncompiled (bytecode)."""
full_name = module.getFullName()
filename = module.getCompileTimeFilename()
if Options.isShowProgress():
inclusion_logger.info("Demoting module %r to bytecode from %r." %
(full_name.asString(), filename))
source_code = module.getSourceCode()
bytecode = demoteSourceCodeToBytecode(module_name=full_name,
source_code=source_code,
filename=filename)
uncompiled_module = makeUncompiledPythonModule(
module_name=full_name,
filename=filename,
bytecode=bytecode,
is_package=module.isCompiledPythonPackage(),
user_provided=True,
technical=False,
)
used_modules = module.getUsedModules()
uncompiled_module.setUsedModules(used_modules)
module.finalize()
if isImportedModuleByName(full_name):
replaceImportedModule(old=module, new=uncompiled_module)
replaceRootModule(old=module, new=uncompiled_module)
from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule
if isTriggerModule(module):
replaceTriggerModule(old=module, new=uncompiled_module)
writeImportedModulesNamesToCache(module_name=full_name,
source_code=source_code,
used_modules=used_modules)
def _detectedSourceFile(filename, module_name, result, user_provided,
technical):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename=filename,
module_name=ModuleName("_collections_abc"),
result=result,
user_provided=user_provided,
technical=technical,
)
source_code = readSourceCodeFromFilename(module_name, filename)
if module_name == "site":
if source_code.startswith("def ") or source_code.startswith("class "):
source_code = "\n" + source_code
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
os.path.basename(filename),
source_code,
)
# Debian stretch site.py
source_code = source_code.replace(
"PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")
# Anaconda3 4.1.2 site.py
source_code = source_code.replace(
"def main():", "def main():return\n\nif 0:\n def _unused():")
if Options.isShowInclusion():
inclusion_logger.info("Freezing module '%s' (from '%s')." %
(module_name, filename))
is_package = os.path.basename(filename) == "__init__.py"
# Plugins can modify source code:
source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
is_package=is_package,
source_code=source_code)
bytecode = compileSourceToBytecode(
source_code=source_code,
filename=module_name.replace(".", os.path.sep) + ".py",
)
# Plugins can modify bytecode code:
bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
is_package=is_package,
bytecode=bytecode)
uncompiled_module = makeUncompiledPythonModule(
module_name=module_name,
bytecode=marshal.dumps(bytecode),
is_package=is_package,
filename=filename,
user_provided=user_provided,
technical=technical,
)
ImportCache.addImportedModule(uncompiled_module)
result.append(uncompiled_module)
module_names.add(module_name)
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points):
# This is terribly complex, because we check the list of used DLLs
# trying to avoid duplicates, and detecting errors with them not
# being binary identical, so we can report them. And then of course
# we also need to handle OS specifics.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
used_dlls = detectUsedDLLs(
source_dir=source_dir,
standalone_entry_points=standalone_entry_points,
use_cache=not Options.shallNotUseDependsExeCachedResults()
and not Options.getWindowsDependencyTool() == "depends.exe",
update_cache=not Options.shallNotStoreDependsExeCachedResults()
and not Options.getWindowsDependencyTool() == "depends.exe",
)
removed_dlls = set()
warned_about = set()
# Fist make checks and remove some.
for dll_filename1, sources1 in tuple(iterItems(used_dlls)):
if dll_filename1 in removed_dlls:
continue
for dll_filename2, sources2 in tuple(iterItems(used_dlls)):
if dll_filename1 == dll_filename2:
continue
if dll_filename2 in removed_dlls:
continue
# Colliding basenames are an issue to us.
if os.path.basename(dll_filename1) != os.path.basename(
dll_filename2):
continue
# May already have been removed earlier
if dll_filename1 not in used_dlls:
continue
if dll_filename2 not in used_dlls:
continue
dll_name = os.path.basename(dll_filename1)
if Options.isShowInclusion():
inclusion_logger.info(
"""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2))
# Check that if a DLL has the same name, if it's identical, then it's easy.
if haveSameFileContents(dll_filename1, dll_filename2):
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
continue
# For Win32 we can check out file versions.
if Utils.isWin32Windows():
dll_version1 = getWindowsDLLVersion(dll_filename1)
dll_version2 = getWindowsDLLVersion(dll_filename2)
if dll_version2 < dll_version1:
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
solved = True
elif dll_version1 < dll_version2:
del used_dlls[dll_filename1]
removed_dlls.add(dll_filename1)
solved = True
else:
solved = False
if solved:
if dll_name not in warned_about and dll_name not in ms_runtime_dlls:
warned_about.add(dll_name)
inclusion_logger.warning(
"Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best."
% dll_name)
continue
# So we have conflicting DLLs, in which case we do report the fact.
inclusion_logger.warning("""\
Ignoring non-identical DLLs for '%s'.
%s used by:
%s
different from
%s used by
%s""" % (
dll_name,
dll_filename1,
"\n ".join(sources1),
dll_filename2,
"\n ".join(sources2),
))
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
dll_map = []
for dll_filename, sources in iterItems(used_dlls):
dll_name = os.path.basename(dll_filename)
target_path = os.path.join(dist_dir, dll_name)
shutil.copyfile(dll_filename, target_path)
dll_map.append((dll_filename, dll_name))
if Options.isShowInclusion():
inclusion_logger.info(
"Included used shared library '%s' (used by %s)." %
(dll_filename, ", ".join(sources)))
if Utils.getOS() == "Darwin":
# For macOS, the binary and the DLLs needs to be changed to reflect
# the relative DLL location in the ".dist" folder.
for standalone_entry_point in standalone_entry_points:
fixupBinaryDLLPathsMacOS(
binary_filename=standalone_entry_point.dest_path,
dll_map=dll_map,
original_location=standalone_entry_point.source_path,
)
for original_path, dll_filename in dll_map:
fixupBinaryDLLPathsMacOS(
binary_filename=os.path.join(dist_dir, dll_filename),
dll_map=dll_map,
original_location=original_path,
)
# Remove code signature from CPython installed library
candidate = os.path.join(
dist_dir,
"Python",
)
if os.path.exists(candidate):
removeMacOSCodeSignature(candidate)
# Remove rpath settings.
if Utils.getOS() in ("Linux", "Darwin"):
# For Linux, the "rpath" of libraries may be an issue and must be
# removed.
if Utils.getOS() == "Darwin":
start = 0
else:
start = 1
for standalone_entry_point in standalone_entry_points[start:]:
removeSharedLibraryRPATH(standalone_entry_point.dest_path)
for _original_path, dll_filename in dll_map:
removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename))
if Utils.isWin32Windows():
if python_version < 0x300:
# For Win32, we might have to remove SXS paths
for standalone_entry_point in standalone_entry_points[1:]:
removeSxsFromDLL(standalone_entry_point.dest_path)
for _original_path, dll_filename in dll_map:
removeSxsFromDLL(os.path.join(dist_dir, dll_filename))
def makeSourceDirectory():
"""Get the full list of modules imported, create code for all of them."""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (
uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()
):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the asked modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneModules():
if module.getFullName() == any_case_module:
break
else:
general.warning(
"Did not follow import to unused '%s', consider include options."
% any_case_module
)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Do some reporting and determine compiled module to work on
compiled_modules = []
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
compiled_modules.append(module)
if Options.isShowInclusion():
inclusion_logger.info(
"Included compiled module '%s'." % module.getFullName()
)
elif module.isPythonExtensionModule():
addExtensionModuleEntryPoint(module)
if Options.isShowInclusion():
inclusion_logger.info(
"Included extension module '%s'." % module.getFullName()
)
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
inclusion_logger.info(
"Included uncompiled module '%s'." % module.getFullName()
)
else:
assert False, module
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=compiled_modules
)
setupProgressBar(
stage="C Source Generation",
unit="module",
total=len(compiled_modules),
)
# Generate code for compiled modules, this can be slow, so do it separately
# with a progress bar.
for module in compiled_modules:
c_filename = module_filenames[module]
reportProgressBar(
item=module.getFullName(),
)
source_code = CodeGeneration.generateModuleCode(
module=module,
data_filename=os.path.basename(c_filename + "onst"), # Really .const
)
writeSourceCode(filename=c_filename, source_code=source_code)
closeProgressBar()
(
helper_decl_code,
helper_impl_code,
constants_header_code,
constants_body_code,
) = CodeGeneration.generateHelpersCode()
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.h"),
source_code=constants_header_code,
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=constants_body_code,
)
def makeSourceDirectory(main_module):
"""Get the full list of modules imported, create code for all of them."""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=too-many-branches
assert main_module.isCompiledPythonModule()
# assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name=module.getFullName(),
module_filename=module.getCompileTimeFilename(),
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if (
uncompiled_module.isUserProvided()
or uncompiled_module.isTechnical()
):
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
if "*" in any_case_module or "{" in any_case_module:
continue
for module in ModuleRegistry.getDoneModules():
if module.getFullName() == any_case_module:
break
else:
general.warning("Not recursing to unused '%s'." % any_case_module)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = OutputDirectories.getSourceDirectoryPath()
module_filenames = pickSourceFilenames(
source_dir=source_dir, modules=ModuleRegistry.getDoneModules()
)
# Generate code for modules.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
c_filename = module_filenames[module]
source_code = CodeGeneration.generateModuleCode(
module=module,
data_filename=os.path.basename(c_filename + "onst"), # Really .const
)
writeSourceCode(filename=c_filename, source_code=source_code)
if Options.isShowInclusion():
inclusion_logger.info(
"Included compiled module '%s'." % module.getFullName()
)
elif module.isPythonShlibModule():
addShlibEntryPoint(module)
if Options.isShowInclusion():
inclusion_logger.info(
"Included extension module '%s'." % module.getFullName()
)
elif module.isUncompiledPythonModule():
if Options.isShowInclusion():
inclusion_logger.info(
"Included uncompiled module '%s'." % module.getFullName()
)
else:
assert False, module
(
helper_decl_code,
helper_impl_code,
constants_header_code,
constants_body_code,
) = CodeGeneration.generateHelpersCode()
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.h"),
source_code=constants_header_code,
)
writeSourceCode(
filename=os.path.join(source_dir, "__constants.c"),
source_code=constants_body_code,
)
for filename, source_code in Plugins.getExtraCodeFiles().items():
target_dir = os.path.join(source_dir, "plugins")
if not os.path.isdir(target_dir):
makePath(target_dir)
writeSourceCode(
filename=os.path.join(target_dir, filename), source_code=source_code
)
def _removeDuplicateDlls(used_dlls):
# Many things to consider, pylint: disable=too-many-branches
removed_dlls = set()
warned_about = set()
# Identical DLLs are interesting for DLL resolution on macOS at least.
duplicate_dlls = {}
# Fist make checks and remove some, in loops we copy the items so we can remove
# the used_dll list freely.
for dll_filename1, (_package_name1,
sources1) in tuple(iterItems(used_dlls)):
if dll_filename1 in removed_dlls:
continue
for dll_filename2, (_package_name1,
sources2) in tuple(iterItems(used_dlls)):
if dll_filename1 == dll_filename2:
continue
if dll_filename2 in removed_dlls:
continue
# Colliding basenames are an issue to us.
if os.path.basename(dll_filename1) != os.path.basename(
dll_filename2):
continue
# May already have been removed earlier
if dll_filename1 not in used_dlls:
continue
if dll_filename2 not in used_dlls:
continue
dll_name = os.path.basename(dll_filename1)
if Options.isShowInclusion():
inclusion_logger.info(
"""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2))
# Check that if a DLL has the same name, if it's identical, then it's easy.
if haveSameFileContents(dll_filename1, dll_filename2):
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
duplicate_dlls.setdefault(dll_filename1,
[]).append(dll_filename2)
duplicate_dlls.setdefault(dll_filename2,
[]).append(dll_filename1)
continue
# For Win32 we can check out file versions.
if Utils.isWin32Windows():
dll_version1 = getWindowsDLLVersion(dll_filename1)
dll_version2 = getWindowsDLLVersion(dll_filename2)
if dll_version2 < dll_version1:
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
solved = True
elif dll_version1 < dll_version2:
del used_dlls[dll_filename1]
removed_dlls.add(dll_filename1)
solved = True
else:
solved = False
if solved:
if dll_name not in warned_about and dll_name not in ms_runtime_dlls:
warned_about.add(dll_name)
inclusion_logger.warning(
"Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best."
% dll_name)
continue
# So we have conflicting DLLs, in which case we do report the fact.
inclusion_logger.warning("""\
Ignoring non-identical DLLs for '%s'.
%s used by:
%s
different from
%s used by
%s""" % (
dll_name,
dll_filename1,
"\n ".join(sources1),
dll_filename2,
"\n ".join(sources2),
))
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
return duplicate_dlls
