def _consider(self, trace_collection, module_filename, module_package):
assert module_package is None or (
type(module_package) is ModuleName
and module_package != ""), repr(module_package)
module_filename = os.path.normpath(module_filename)
module_name, module_kind = getModuleNameAndKindFromFilename(
module_filename)
if module_kind is not None:
module_fullpath = ModuleName.makeModuleNameInPackage(
module_name, module_package)
decision, reason = decideRecursion(
module_filename=module_filename,
module_name=module_fullpath,
module_kind=module_kind,
)
if decision:
module_relpath = relpath(module_filename)
imported_module, added_flag = recurseTo(
module_package=module_package,
module_filename=module_filename,
module_relpath=module_relpath,
module_kind=module_kind,
reason=reason,
)
if added_flag:
trace_collection.signalChange(
"new_code",
imported_module.getSourceReference(),
"Recursed to module.",
)
return imported_module
elif decision is False and module_kind == "py":
uncompiled_module = getUncompiledModule(
module_fullpath, module_filename)
if uncompiled_module is not None:
return uncompiled_module
elif decision is None and module_kind == "py":
if (module_filename not in self._warned_about
and module_fullpath not in getModuleIgnoreList()):
self._warned_about.add(module_filename)
inclusion_logger.warning("""\
Not recursing to '%(full_path)s' (%(filename)s), please specify \
--nofollow-imports (do not warn), \
--follow-imports (recurse to all), \
--nofollow-import-to=%(full_path)s (ignore it), \
--follow-import-to=%(full_path)s (recurse to it) to change.""" % {
"full_path": module_fullpath,
"filename": module_filename
})
def copyDataFiles(dist_dir):
"""Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
# Many details to deal with, pylint: disable=too-many-branches,too-many-locals
for pattern, dest, arg in Options.getShallIncludeDataFiles():
filenames = resolveShellPatternToFilenames(pattern)
if not filenames:
inclusion_logger.warning("No match data file to be included: %r" %
pattern)
for filename in filenames:
file_reason = "specified data file %r on command line" % arg
rel_path = dest
if rel_path.endswith(("/", os.path.sep)):
rel_path = os.path.join(rel_path, os.path.basename(filename))
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
for src, dest in Options.getShallIncludeDataDirs():
filenames = getFileList(src)
if not filenames:
inclusion_logger.warning("No files in directory" % src)
for filename in filenames:
relative_filename = relpath(filename, src)
file_reason = "specified data dir %r on command line" % src
rel_path = os.path.join(dest, relative_filename)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
for plugin, included_datafile in Plugins.considerDataFiles(module):
_handleDataFile(dist_dir=dist_dir,
tracer=plugin,
included_datafile=included_datafile)
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonPackage(
) or module.isUncompiledPythonPackage():
package_name = module.getFullName()
match, reason = package_name.matchesToShellPatterns(
patterns=Options.getShallIncludePackageData())
if match:
package_directory = module.getCompileTimeDirectory()
pkg_filenames = getFileList(
package_directory,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
getSharedLibrarySuffixes(),
)
if pkg_filenames:
file_reason = "package '%s' %s" % (package_name, reason)
for pkg_filename in pkg_filenames:
rel_path = os.path.join(
package_name.asPath(),
os.path.relpath(pkg_filename, package_directory),
)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(pkg_filename, rel_path,
file_reason),
)
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points):
# This is terribly complex, because we check the list of used DLLs
# trying to avoid duplicates, and detecting errors with them not
# being binary identical, so we can report them. And then of course
# we also need to handle OS specifics.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
used_dlls = detectUsedDLLs(
source_dir=source_dir,
standalone_entry_points=standalone_entry_points,
use_cache=not Options.shallNotUseDependsExeCachedResults()
and not Options.getWindowsDependencyTool() == "depends.exe",
update_cache=not Options.shallNotStoreDependsExeCachedResults()
and not Options.getWindowsDependencyTool() == "depends.exe",
)
removed_dlls = set()
warned_about = set()
# Fist make checks and remove some.
for dll_filename1, sources1 in tuple(iterItems(used_dlls)):
if dll_filename1 in removed_dlls:
continue
for dll_filename2, sources2 in tuple(iterItems(used_dlls)):
if dll_filename1 == dll_filename2:
continue
if dll_filename2 in removed_dlls:
continue
# Colliding basenames are an issue to us.
if os.path.basename(dll_filename1) != os.path.basename(
dll_filename2):
continue
# May already have been removed earlier
if dll_filename1 not in used_dlls:
continue
if dll_filename2 not in used_dlls:
continue
dll_name = os.path.basename(dll_filename1)
if Options.isShowInclusion():
inclusion_logger.info(
"""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2))
# Check that if a DLL has the same name, if it's identical, then it's easy.
if haveSameFileContents(dll_filename1, dll_filename2):
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
continue
# For Win32 we can check out file versions.
if Utils.isWin32Windows():
dll_version1 = getWindowsDLLVersion(dll_filename1)
dll_version2 = getWindowsDLLVersion(dll_filename2)
if dll_version2 < dll_version1:
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
solved = True
elif dll_version1 < dll_version2:
del used_dlls[dll_filename1]
removed_dlls.add(dll_filename1)
solved = True
else:
solved = False
if solved:
if dll_name not in warned_about and dll_name not in ms_runtime_dlls:
warned_about.add(dll_name)
inclusion_logger.warning(
"Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best."
% dll_name)
continue
# So we have conflicting DLLs, in which case we do report the fact.
inclusion_logger.warning("""\
Ignoring non-identical DLLs for '%s'.
%s used by:
%s
different from
%s used by
%s""" % (
dll_name,
dll_filename1,
"\n ".join(sources1),
dll_filename2,
"\n ".join(sources2),
))
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
dll_map = []
for dll_filename, sources in iterItems(used_dlls):
dll_name = os.path.basename(dll_filename)
target_path = os.path.join(dist_dir, dll_name)
shutil.copyfile(dll_filename, target_path)
dll_map.append((dll_filename, dll_name))
if Options.isShowInclusion():
inclusion_logger.info(
"Included used shared library '%s' (used by %s)." %
(dll_filename, ", ".join(sources)))
if Utils.getOS() == "Darwin":
# For macOS, the binary and the DLLs needs to be changed to reflect
# the relative DLL location in the ".dist" folder.
for standalone_entry_point in standalone_entry_points:
fixupBinaryDLLPathsMacOS(
binary_filename=standalone_entry_point.dest_path,
dll_map=dll_map,
original_location=standalone_entry_point.source_path,
)
for original_path, dll_filename in dll_map:
fixupBinaryDLLPathsMacOS(
binary_filename=os.path.join(dist_dir, dll_filename),
dll_map=dll_map,
original_location=original_path,
)
# Remove code signature from CPython installed library
candidate = os.path.join(
dist_dir,
"Python",
)
if os.path.exists(candidate):
removeMacOSCodeSignature(candidate)
# Remove rpath settings.
if Utils.getOS() in ("Linux", "Darwin"):
# For Linux, the "rpath" of libraries may be an issue and must be
# removed.
if Utils.getOS() == "Darwin":
start = 0
else:
start = 1
for standalone_entry_point in standalone_entry_points[start:]:
removeSharedLibraryRPATH(standalone_entry_point.dest_path)
for _original_path, dll_filename in dll_map:
removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename))
if Utils.isWin32Windows():
if python_version < 0x300:
# For Win32, we might have to remove SXS paths
for standalone_entry_point in standalone_entry_points[1:]:
removeSxsFromDLL(standalone_entry_point.dest_path)
for _original_path, dll_filename in dll_map:
removeSxsFromDLL(os.path.join(dist_dir, dll_filename))
def _removeDuplicateDlls(used_dlls):
# Many things to consider, pylint: disable=too-many-branches
removed_dlls = set()
warned_about = set()
# Identical DLLs are interesting for DLL resolution on macOS at least.
duplicate_dlls = {}
# Fist make checks and remove some, in loops we copy the items so we can remove
# the used_dll list freely.
for dll_filename1, (_package_name1,
sources1) in tuple(iterItems(used_dlls)):
if dll_filename1 in removed_dlls:
continue
for dll_filename2, (_package_name1,
sources2) in tuple(iterItems(used_dlls)):
if dll_filename1 == dll_filename2:
continue
if dll_filename2 in removed_dlls:
continue
# Colliding basenames are an issue to us.
if os.path.basename(dll_filename1) != os.path.basename(
dll_filename2):
continue
# May already have been removed earlier
if dll_filename1 not in used_dlls:
continue
if dll_filename2 not in used_dlls:
continue
dll_name = os.path.basename(dll_filename1)
if Options.isShowInclusion():
inclusion_logger.info(
"""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2))
# Check that if a DLL has the same name, if it's identical, then it's easy.
if haveSameFileContents(dll_filename1, dll_filename2):
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
duplicate_dlls.setdefault(dll_filename1,
[]).append(dll_filename2)
duplicate_dlls.setdefault(dll_filename2,
[]).append(dll_filename1)
continue
# For Win32 we can check out file versions.
if Utils.isWin32Windows():
dll_version1 = getWindowsDLLVersion(dll_filename1)
dll_version2 = getWindowsDLLVersion(dll_filename2)
if dll_version2 < dll_version1:
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
solved = True
elif dll_version1 < dll_version2:
del used_dlls[dll_filename1]
removed_dlls.add(dll_filename1)
solved = True
else:
solved = False
if solved:
if dll_name not in warned_about and dll_name not in ms_runtime_dlls:
warned_about.add(dll_name)
inclusion_logger.warning(
"Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best."
% dll_name)
continue
# So we have conflicting DLLs, in which case we do report the fact.
inclusion_logger.warning("""\
Ignoring non-identical DLLs for '%s'.
%s used by:
%s
different from
%s used by
%s""" % (
dll_name,
dll_filename1,
"\n ".join(sources1),
dll_filename2,
"\n ".join(sources2),
))
del used_dlls[dll_filename2]
removed_dlls.add(dll_filename2)
return duplicate_dlls