def _attemptRecursion(self, trace_collection):
found = False
parent_module = self.getParentModule()
if parent_module.isCompiledPythonPackage():
parent_package = parent_module.getFullName()
else:
parent_package = self.getParentModule().getPackage()
module_package, module_filename, self.finding = findModule(
importing = self,
module_name = self.getModuleName(),
parent_package = parent_package,
level = self.getLevel(),
warn = True
)
if module_filename is not None:
self.imported_module = self._consider(
trace_collection = trace_collection,
module_filename = module_filename,
module_package = module_package
)
if self.imported_module is not None:
found = self.imported_module.getFullName()
self.found_modules = []
import_list = self.getImportList()
if import_list and \
self.imported_module.isCompiledPythonPackage():
for import_item in import_list:
if import_item == '*':
continue
module_package, module_filename, _finding = findModule(
importing = self,
module_name = import_item,
parent_package = found,
level = -1,
warn = False
)
if module_filename is not None:
sub_imported_module = self._consider(
trace_collection = trace_collection,
module_filename = module_filename,
module_package = module_package
)
if sub_imported_module is not None:
self.found_modules.append(
sub_imported_module.getFullName()
)
return found
def _attemptRecursion(self, constraint_collection):
found = False
parent_module = self.getParentModule()
if parent_module.isCompiledPythonPackage():
parent_package = parent_module.getFullName()
else:
parent_package = self.getParentModule().getPackage()
module_package, module_filename, _finding = findModule(
importing = self,
module_name = self.getModuleName(),
parent_package = parent_package,
level = self.getLevel(),
warn = True
)
if module_filename is not None:
imported_module = self._consider(
constraint_collection = constraint_collection,
module_filename = module_filename,
module_package = module_package
)
if imported_module is not None:
found = imported_module.getFullName()
self.found_modules = []
import_list = self.getImportList()
if import_list and imported_module.isCompiledPythonPackage():
for import_item in import_list:
if import_item == '*':
continue
module_package, module_filename, _finding = findModule(
importing = self,
module_name = import_item,
parent_package = imported_module.getFullName(),
level = -1,
warn = False
)
if module_filename is not None:
sub_imported_module = self._consider(
constraint_collection = constraint_collection,
module_filename = module_filename,
module_package = module_package
)
if sub_imported_module is not None:
self.found_modules.append(sub_imported_module.getFullName())
return found
def _attemptRecursion(self, constraint_collection):
assert self.getModule() is None
parent_module = self.getParentModule()
if parent_module.isPythonPackage():
parent_package = parent_module.getFullName()
else:
parent_package = self.getParentModule().getPackage()
module_package, module_filename, _finding = findModule(
source_ref = self.source_ref,
module_name = self.getModuleName(),
parent_package = parent_package,
level = self.getLevel(),
warn = True
)
if module_filename is not None:
imported_module = self._consider(
constraint_collection = constraint_collection,
module_filename = module_filename,
module_package = module_package
)
if imported_module is not None:
self.setModule(imported_module)
self.found_modules = []
import_list = self.getImportList()
if import_list and imported_module.isPythonPackage():
for import_item in import_list:
if import_item == '*':
continue
module_package, module_filename, _finding = findModule(
source_ref = self.source_ref,
module_name = import_item,
parent_package = imported_module.getFullName(),
level = -1,
warn = False
)
if module_filename is not None:
sub_imported_module = self._consider(
constraint_collection = constraint_collection,
module_filename = module_filename,
module_package = module_package
)
if sub_imported_module is not None:
self.found_modules.append(sub_imported_module)
def attemptRecursion(self):
# Make sure the package is recursed to.
# Return the list of newly added modules.
result = []
if self.package_name is not None and self.package is None:
self.package = getModuleByName(self.package_name)
if self.package_name is not None and self.package is None:
package_package, package_filename, finding = findModule(
importing=self,
module_name=self.package_name,
parent_package=None,
level=1,
warn=python_version < 330)
# TODO: Temporary, if we can't find the package for Python3.3 that
# is semi-OK, maybe.
if python_version >= 330 and not package_filename:
return []
if self.package_name == "uniconvertor.app.modules":
return []
assert package_filename is not None, (self.package_name, finding)
_package_name, package_kind = getModuleNameAndKindFromFilename(
package_filename)
# assert _package_name == self.package_name, (package_filename, _package_name, self.package_name)
decision, _reason = decideRecursion(
module_filename=package_filename,
module_name=self.package_name,
module_package=package_package,
module_kind=package_kind)
if decision is not None:
self.package, is_added = recurseTo(
module_package=package_package,
module_filename=package_filename,
module_relpath=relpath(package_filename),
module_kind="py",
reason="Containing package of recursed module '%s'." %
self.getFullName(),
)
if is_added:
result.append(self.package)
if self.package:
from nuitka.ModuleRegistry import addUsedModule
addUsedModule(self.package)
# print "Recursed to package", self.package_name
result.extend(self.package.attemptRecursion())
return result
def attemptRecursion(self):
# Make sure the package is recursed to.
# Return the list of newly added modules.
result = []
if self.package_name is not None and self.package is None:
package_package, package_filename, _finding = findModule(
importing = self,
module_name = self.package_name,
parent_package = None,
level = 1,
warn = python_version < 330
)
# TODO: Temporary, if we can't find the package for Python3.3 that
# is semi-OK, maybe.
if python_version >= 330 and not package_filename:
return []
if self.package_name == "uniconvertor.app.modules":
return []
assert package_filename is not None, self.package_name
_package_name, package_kind = getModuleNameAndKindFromFilename(package_filename)
# assert _package_name == self.package_name, (package_filename, _package_name, self.package_name)
decision, _reason = decideRecursion(
module_filename = package_filename,
module_name = self.package_name,
module_package = package_package,
module_kind = package_kind
)
if decision is not None:
imported_module, is_added = recurseTo(
module_package = package_package,
module_filename = package_filename,
module_relpath = Utils.relpath(package_filename),
module_kind = "py",
reason = "Containing package of recursed module '%s'." % self.getFullName(),
)
self.package = imported_module
if is_added:
result.append(imported_module)
if self.package:
from nuitka.ModuleRegistry import addUsedModule
addUsedModule(self.package)
# print "Recursed to package", self.package_name
result.extend(self.package.attemptRecursion())
return result
def getScanDirectories(package_name, original_dir):
cache_key = package_name, original_dir
if cache_key in _scan_dir_cache:
return _scan_dir_cache[cache_key]
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
for path_dir in os.environ["PATH"].split(";"):
if not os.path.isdir(path_dir):
continue
if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "System32")):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
continue
scan_dirs.append(path_dir)
result = []
# Remove directories that hold no DLLs.
for scan_dir in scan_dirs:
sys.stdout.flush()
# These are useless, but plenty.
if os.path.basename(scan_dir) == "__pycache__":
continue
# No DLLs, no use.
if not any(entry[1].lower().endswith(".dll")
for entry in listDir(scan_dir)):
continue
result.append(scan_dir)
_scan_dir_cache[cache_key] = result
return result
def _buildPackage(self, build_lib):
# Nuitka wants the main package by filename, probably we should stop
# needing that.
from nuitka.importing.Importing import findModule, setMainScriptDirectory
from nuitka.utils.Execution import getExecutablePath
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferrably.
setMainScriptDirectory('.')
package, main_filename, finding = findModule(
importing=None,
module_name=self.main_package,
parent_package=None,
level=0,
warn=False)
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
assert package is None, package
recurse_packages = self.compile_packages.copy()
if os.path.isdir(self.main_package):
# Include all python files in wheel
for root, dirs, files in os.walk(self.main_package):
for fn in fnmatch.filter(files, '*.py'):
module = os.path.join(
root, fn)[:-3] if fn != '__init__.py' else root
pypath = ".".join(module.split(os.sep))
recurse_packages.append(pypath)
command = [
sys.executable,
"-m",
"nuitka",
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % build_lib,
"--recurse-dir=%s" % self.main_package,
"--recurse-not-to=*.tests",
"--show-modules",
"--remove-output",
main_filename,
] + ["--recurse-to=%s" % p for p in recurse_packages]
subprocess.check_call(command)
os.chdir(old_dir)
self.build_lib = build_lib
def getPackageSpecificDLLDirectories(package_name):
scan_dirs = OrderedSet()
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.add(package_dir)
scan_dirs.update(
getSubDirectories(package_dir, ignore_dirs=("__pycache__", )))
scan_dirs.update(Plugins.getModuleSpecificDllPaths(package_name))
return scan_dirs
def _buildPackage(self, build_lib):
# Nuitka wants the main package by filename, probably we should stop
# needing that.
from nuitka.importing.Importing import findModule, setMainScriptDirectory
from nuitka.utils.Execution import getExecutablePath
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferrably.
setMainScriptDirectory('.')
package, main_filename, finding = findModule(
importing=None,
module_name=self.main_package,
parent_package=None,
level=0,
warn=False)
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
assert package is None, package
nuitka_binary = getExecutablePath("nuitka")
if nuitka_binary is None:
sys.exit("Error, cannot find nuitka binary in PATH.")
command = [
sys.executable,
nuitka_binary,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % build_lib,
"--recurse-to={%s}" % ','.join(self.compile_packages),
"--recurse-dir=%s" % self.main_package,
"--recurse-not-to=*.tests",
"--show-modules",
"--remove-output",
main_filename,
]
subprocess.check_call(command)
os.chdir(old_dir)
self.build_lib = build_lib
def _attemptRecursion(self, trace_collection, module_name):
# Complex stuff, pylint: disable=too-many-branches
parent_module = self.getParentModule()
if parent_module.isCompiledPythonPackage():
parent_package = parent_module.getFullName()
else:
parent_package = self.getParentModule().getPackage()
level = self.getLevel()
if level is None:
level = 0 if parent_module.getFutureSpec().isAbsoluteImport() else -1
elif not level.isCompileTimeConstant():
return
else:
level = level.getCompileTimeConstant()
# TODO: Catch this as a static error maybe.
if type(level) not in (int, long):
return
module_package, module_filename, self.finding = findModule(
importing=self,
module_name=module_name,
parent_package=parent_package,
level=level,
warn=True,
)
if module_filename is not None:
imported_module = self._consider(
trace_collection=trace_collection,
module_filename=module_filename,
module_package=module_package,
)
if imported_module is not None:
self.imported_module_desc = (
imported_module.getFullName(),
imported_module.getFilename(),
)
import_list = self.getFromList()
if import_list is not None:
if import_list.isCompileTimeConstant():
import_list = import_list.getCompileTimeConstant()
if type(import_list) not in (tuple, list):
import_list = None
if import_list and imported_module.isCompiledPythonPackage():
for import_item in import_list:
if import_item == "*":
continue
module_package, module_filename, _finding = findModule(
importing=self,
module_name=import_item,
parent_package=imported_module.getFullName(),
level=-1, # Relative import, so child is used.
warn=False,
)
if module_filename is not None:
sub_imported_module = self._consider(
trace_collection=trace_collection,
module_filename=module_filename,
module_package=module_package,
)
if sub_imported_module is not None:
self.import_list_modules_desc.append(
(
sub_imported_module.getFullName(),
sub_imported_module.getFilename(),
)
)
else:
while "." in module_name:
module_name = ".".join(module_name.split(".")[:-1])
module_package, module_filename, _finding = findModule(
importing=self,
module_name=module_name,
parent_package=parent_package,
level=level,
warn=True,
)
if module_filename is not None:
package_module = self._consider(
trace_collection=trace_collection,
module_filename=module_filename,
module_package=module_package,
)
if package_module is not None:
if self.package_modules_desc is None:
self.package_modules_desc = []
self.package_modules_desc.append(
(package_module.getFullName(), package_module.getFilename())
)
def _build(self, build_lib):
# High complexity, pylint: disable=too-many-branches,too-many-locals
# Nuitka wants the main package by filename, probably we should stop
# needing that.
from nuitka.importing.Importing import findModule, setMainScriptDirectory
from nuitka.utils.ModuleNames import ModuleName
from nuitka.__past__ import ( # pylint: disable=I0021,redefined-builtin
Iterable,
unicode,
)
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferably.
setMainScriptDirectory(".")
to_builds = self._find_to_build()
for to_build in to_builds:
package, main_filename, finding = findModule(
importing=None,
module_name=ModuleName(to_build.module_name),
parent_package=None,
level=0,
warn=False,
)
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
if package is not None:
output_dir = os.path.join(build_lib, package)
else:
output_dir = build_lib
command = [
sys.executable,
"-m",
"nuitka",
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % output_dir,
"--nofollow-import-to=*.tests",
"--show-modules",
"--remove-output",
]
if type(to_build) is PyPackage:
command += [
"--include-package=%s" % package_name.replace("/", ".")
for package_name in to_build.related_packages
]
else: # type(to_build) is PyModule
command += [
"--include-module=%s" % module_name
for module_name in to_build.related_modules
]
# Process any extra options from setuptools
if "nuitka" in self.distribution.command_options:
for option, value in self.distribution.command_options[
"nuitka"
].items():
option = "--" + option.lstrip("-")
if value is None:
command.append(option)
elif isinstance(value, bool):
option = "--" + ("no" if not value else "") + option.lstrip("-")
command.append(option)
elif isinstance(value, Iterable) and not isinstance(
value, (unicode, bytes, str)
):
for val in value:
command.append("%s=%s" % (option, val))
else:
command.append("%s=%s" % (option, value))
command.append(main_filename)
# added for clarity
my_print("Building: %s" % to_build, style="yellow")
subprocess.check_call(command, cwd=build_lib)
for root, _, filenames in os.walk(build_lib):
for filename in filenames:
fullpath = os.path.join(root, filename)
if fullpath.lower().endswith((".py", ".pyw", ".pyc", ".pyo")):
os.unlink(fullpath)
os.chdir(old_dir)
self.build_lib = build_lib
def _attemptRecursion(self, trace_collection, module_name):
# Complex stuff, pylint: disable=too-many-branches
parent_module = self.getParentModule()
parent_package = parent_module.getFullName()
if not parent_module.isCompiledPythonPackage():
parent_package = parent_package.getPackageName()
level = self.subnode_level
if level is None:
level = 0 if parent_module.getFutureSpec().isAbsoluteImport(
) else -1
elif not level.isCompileTimeConstant():
return
else:
level = level.getCompileTimeConstant()
# TODO: Catch this as a static error maybe.
if type(level) not in (int, long):
return
module_package, self.module_filename, self.finding = findModule(
importing=self,
module_name=ModuleName(module_name),
parent_package=parent_package,
level=level,
warn=True,
)
if self.module_filename is not None:
imported_module = self._consider(
trace_collection=trace_collection,
module_filename=self.module_filename,
module_package=module_package,
)
if imported_module is not None:
self.imported_module_desc = (
imported_module.getFullName(),
imported_module.getFilename(),
)
import_list = self.subnode_fromlist
if import_list is not None:
if import_list.isCompileTimeConstant():
import_list = import_list.getCompileTimeConstant()
if type(import_list) not in (tuple, list):
import_list = None
if import_list and imported_module.isCompiledPythonPackage():
for import_item in import_list:
if import_item == "*":
continue
module_package, module_filename, _finding = findModule(
importing=self,
module_name=ModuleName(import_item),
parent_package=imported_module.getFullName(),
level=-1, # Relative import, so child is used.
warn=False,
)
if module_filename is not None:
sub_imported_module = self._consider(
trace_collection=trace_collection,
module_filename=module_filename,
module_package=module_package,
)
if sub_imported_module is not None:
self.import_list_modules_desc.append((
sub_imported_module.getFullName(),
sub_imported_module.getFilename(),
))
else:
module_name = ModuleName(module_name)
while True:
module_name = module_name.getPackageName()
if module_name is None:
break
module_package, module_filename, _finding = findModule(
importing=self,
module_name=module_name,
parent_package=parent_package,
level=level,
warn=True,
)
if module_filename is not None:
package_module = self._consider(
trace_collection=trace_collection,
module_filename=module_filename,
module_package=module_package,
)
if package_module is not None:
if self.package_modules_desc is None:
self.package_modules_desc = []
self.package_modules_desc.append(
(package_module.getFullName(),
package_module.getFilename()))
def _detectBinaryPathDLLsWindows(is_main_executable, source_dir, original_dir,
binary_filename, package_name):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-locals
result = set()
cache_filename = _getCacheFilename(is_main_executable, source_dir,
original_dir, binary_filename)
if os.path.exists(cache_filename
) and not Options.shallNotUseDependsExeCachedResults():
for line in open(cache_filename):
line = line.strip()
result.add(line)
return result
# User query should only happen once if at all.
with _withLock():
depends_exe = getDependsExePath()
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
with _withLock():
# The search order by default prefers the system directory, where a
# wrong "PythonXX.dll" might be living.
with open(binary_filename + ".dwp", 'w') as dwp_file:
dwp_file.write(
"""\
KnownDLLs
%(original_dirs)s
SysPath
32BitSysDir
16BitSysDir
OSDir
AppPath
SxS
""" % {
"original_dirs":
'\n'.join("UserDir %s" % dirname for dirname in scan_dirs
if not os.path.basename(dirname) == "__pycache__"
if any(entry[1].lower().endswith(".dll")
for entry in listDir(dirname))),
})
# Starting the process while locked, so file handles are not duplicated.
depends_exe_process = subprocess.Popen(
(depends_exe, "-c", "-ot%s" % binary_filename + ".depends",
"-d:%s" % binary_filename + ".dwp", "-f1", "-pa1", "-ps1",
binary_filename))
# TODO: Exit code should be checked.
depends_exe_process.wait()
# Opening the result under lock, so it is not getting locked by new processes.
with _withLock():
_parseDependsExeOutput(binary_filename + ".depends", result)
deleteFile(binary_filename + ".depends", must_exist=True)
deleteFile(binary_filename + ".dwp", must_exist=True)
if not Options.shallNotStoreDependsExeCachedResults():
with open(cache_filename, 'w') as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def _buildPackage(self, build_lib):
# High complexity, pylint: disable=too-many-branches,too-many-locals
# Nuitka wants the main package by filename, probably we should stop
# needing that.
from nuitka.importing.Importing import findModule, setMainScriptDirectory
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferably.
setMainScriptDirectory(".")
package, main_filename, finding = findModule(
importing=None,
module_name=self.main_package,
parent_package=None,
level=0,
warn=False,
)
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
assert package is None, package
# If there are other files left over in the wheel after python scripts
# are compiled, we'll keep the folder structure with the files in the wheel
keep_resources = False
python_files = []
if os.path.isdir(main_filename):
# Include all python files in wheel
for root, dirs, files in os.walk(main_filename):
if "__pycache__" in dirs:
dirs.remove("__pycache__")
shutil.rmtree(os.path.join(root, "__pycache__"))
for fn in files:
if fn.lower().endswith((".py", ".pyc", ".pyo")):
# These files will definitely be deleted once nuitka
# has compiled the main_package
python_files.append(os.path.join(root, fn))
else:
keep_resources = True
output_dir = build_lib
command = [
sys.executable,
"-m",
"nuitka",
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % output_dir,
"--include-package=%s" % self.main_package,
"--nofollow-import-to=*.tests",
"--show-modules",
"--remove-output",
]
# Process any extra options from setuptools
if "nuitka" in self.distribution.command_options:
for option, details in self.distribution.command_options["nuitka"].items():
option = "--" + option.lstrip("-")
_source, value = details
if value is None:
command.append(option)
elif isinstance(value, collections.Iterable) and not isinstance(
value, (unicode, bytes, str)
):
for val in value:
command.append("%s=%s" % (option, val))
else:
command.append("%s=%s" % (option, value))
command.append(main_filename)
subprocess.check_call(command, cwd=build_lib)
os.chdir(old_dir)
self.build_lib = build_lib
if keep_resources:
# Delete the individual source files
for fn in python_files:
os.unlink(fn)
else:
# Delete the entire source copy of the module
shutil.rmtree(os.path.join(self.build_lib, self.main_package))
def _detectBinaryPathDLLsWindowsPE(is_main_executable, source_dir,
original_dir, binary_filename,
package_name):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-branches,too-many-locals
result = set()
cache_filename = _getCacheFilename(is_main_executable, source_dir,
original_dir, binary_filename)
if (os.path.exists(cache_filename)
and not Options.shallNotUseDependsExeCachedResults()):
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if os.path.isdir(original_dir):
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
if Options.isExperimental("use_pefile_fullrecurse"):
try:
scan_dirs.extend(getSubDirectories(get_python_lib()))
except OSError:
print(
"Cannot recurse into site-packages for dependencies. Path not found."
)
else:
# Fix for missing pywin32 inclusion when using pythonwin library, no way to detect that automagically
# Since there are more than one dependencies on pywintypes37.dll, let's include this anyway
# In recursive mode, using dirname(original_dir) won't always work, hence get_python_lib
try:
scan_dirs.append(os.path.join(get_python_lib(),
"pywin32_system32"))
except OSError:
pass
# Add native system directory based on pe file architecture and os architecture
# Python 32: system32 = syswow64 = 32 bits systemdirectory
# Python 64: system32 = 64 bits systemdirectory, syswow64 = 32 bits systemdirectory
binary_file_is_64bit = _isPE64(binary_filename)
python_is_64bit = getArchitecture() == "x86_64"
if binary_file_is_64bit is not python_is_64bit:
print("Warning: Using Python x64=%s with x64=%s binary dependencies" %
(binary_file_is_64bit, python_is_64bit))
if binary_file_is_64bit:
# This is actually not useful as of today since we don't compile 32 bits on 64 bits
if python_is_64bit:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
"System32"))
else:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"],
"SysWOW64"))
else:
scan_dirs.append(os.path.join(os.environ["SYSTEMROOT"], "System32"))
if Options.isExperimental("use_pefile_recurse"):
# Recursive one level scanning of all .pyd and .dll in the original_dir too
# This shall fix a massive list of missing dependencies that may come with included libraries which themselves
# need to be scanned for inclusions
for root, _, filenames in os.walk(original_dir):
for optional_libary in filenames:
if optional_libary.endswith(
".dll") or optional_libary.endswith(".pyd"):
_parsePEFileOutput(os.path.join(root, optional_libary),
scan_dirs, result)
_parsePEFileOutput(binary_filename, scan_dirs, result)
if not Options.shallNotStoreDependsExeCachedResults():
with open(cache_filename, "w") as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def _attemptRecursion(self, trace_collection, module_name):
# Complex stuff, pylint: disable=too-many-branches
parent_module = self.getParentModule()
if parent_module.isCompiledPythonPackage():
parent_package = parent_module.getFullName()
else:
parent_package = self.getParentModule().getPackage()
level = self.getLevel()
if level is None:
level = 0 if parent_module.getFutureSpec().isAbsoluteImport() else -1
elif not level.isCompileTimeConstant():
return
else:
level = level.getCompileTimeConstant()
# TODO: Catch this as a static error maybe.
if type(level) not in (int, long):
return
module_package, module_filename, self.finding = findModule(
importing = self,
module_name = module_name,
parent_package = parent_package,
level = level,
warn = True
)
if module_filename is not None:
self.imported_module = self._consider(
trace_collection = trace_collection,
module_filename = module_filename,
module_package = module_package
)
if self.imported_module is not None:
import_list = self.getFromList()
if import_list is not None:
if import_list.isCompileTimeConstant():
import_list = import_list.getCompileTimeConstant()
if type(import_list) not in (tuple, list):
import_list = None
if import_list and \
self.imported_module.isCompiledPythonPackage():
for import_item in import_list:
if import_item == '*':
continue
module_package, module_filename, _finding = findModule(
importing = self,
module_name = import_item,
parent_package = self.imported_module.getFullName(),
level = -1, # Relative import, so child is used.
warn = False
)
if module_filename is not None:
sub_imported_module = self._consider(
trace_collection = trace_collection,
module_filename = module_filename,
module_package = module_package
)
if sub_imported_module is not None:
self.import_list_modules.append(
sub_imported_module.getFullName()
)
else:
while "." in module_name:
module_name = ".".join(module_name.split(".")[:-1])
module_package, module_filename, _finding = findModule(
importing = self,
module_name = module_name,
parent_package = parent_package,
level = level,
warn = True
)
if module_filename is not None:
package_module = self._consider(
trace_collection = trace_collection,
module_filename = module_filename,
module_package = module_package
)
if package_module is not None:
if self.package_modules is None:
self.package_modules = []
self.package_modules.append(package_module)
def getScanDirectories(package_name, original_dir):
# Many cases, pylint: disable=too-many-branches
cache_key = package_name, original_dir
if cache_key in _scan_dir_cache:
return _scan_dir_cache[cache_key]
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
if (Utils.isWin32Windows() and package_name is not None
and package_name.isBelowNamespace("win32com")):
pywin32_dir = getPyWin32Dir()
if pywin32_dir is not None:
scan_dirs.append(pywin32_dir)
for path_dir in os.environ["PATH"].split(";"):
if not os.path.isdir(path_dir):
continue
if areSamePaths(path_dir, os.path.join(os.environ["SYSTEMROOT"])):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "System32")):
continue
if areSamePaths(path_dir,
os.path.join(os.environ["SYSTEMROOT"], "SysWOW64")):
continue
scan_dirs.append(path_dir)
result = []
# Remove directories that hold no DLLs.
for scan_dir in scan_dirs:
sys.stdout.flush()
# These are useless, but plenty.
if os.path.basename(scan_dir) == "__pycache__":
continue
scan_dir = getDirectoryRealPath(scan_dir)
# No DLLs, no use.
if not any(entry[1].lower().endswith(".dll")
for entry in listDir(scan_dir)):
continue
result.append(os.path.realpath(scan_dir))
_scan_dir_cache[cache_key] = result
return result
def _buildPackage(self, build_lib):
# Nuitka wants the main package by filename, probably we should stop
# needing that, pylint: disable=too-many-locals
from nuitka.importing.Importing import findModule, setMainScriptDirectory
old_dir = os.getcwd()
os.chdir(build_lib)
# Search in the build directory preferably.
setMainScriptDirectory('.')
package, main_filename, finding = findModule(
importing = None,
module_name = self.main_package,
parent_package = None,
level = 0,
warn = False
)
# Check expectations, e.g. do not compile built-in modules.
assert finding == "absolute", finding
assert package is None, package
# If there are other files left over in the wheel after python scripts
# are compiled, we'll keep the folder structure with the files in the wheel
keep_resources = False
python_files = []
if os.path.isdir(main_filename):
# Include all python files in wheel
for root, dirs, files in os.walk(main_filename):
if "__pycache__" in dirs:
dirs.remove("__pycache__")
shutil.rmtree(os.path.join(root, "__pycache__"))
for fn in files:
if fn.lower().endswith((".py", ".pyc", ".pyo")):
# These files will definitely be deleted once nuitka
# has compiled the main_package
python_files.append(os.path.join(root, fn))
else:
keep_resources = True
output_dir = build_lib
command = [
sys.executable,
"-m", "nuitka",
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % output_dir,
"--recurse-dir=%s" % self.main_package,
"--recurse-to=%s" % self.main_package,
"--recurse-not-to=*.tests",
"--show-modules",
"--remove-output",
main_filename,
]
subprocess.check_call(
command
)
os.chdir(old_dir)
self.build_lib = build_lib
if keep_resources:
# Delete the individual source files
for fn in python_files:
os.unlink(fn)
else:
# Delete the entire source copy of the module
shutil.rmtree(os.path.join(self.build_lib, self.main_package))