def _get_subdir_files(module, subdirs, folders_only):
"""Yield filenames in given subdirs of the module.
Notes:
All filenames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - str or None or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest) are yielded if folders_only is False,
else tuples (_createEmptyDirNone, dest) are yielded.
"""
module_folder = module.getCompileTimeDirectory()
elements = module.getFullName().split(".")
filename_start = module_folder.find(elements[0])
file_list = []
item_set = OrderedSet()
if subdirs is None:
file_list = getFileList(module_folder)
elif type(subdirs) is str:
data_dir = os.path.join(module_folder, subdirs)
file_list = getFileList(data_dir)
else:
for subdir in subdirs:
data_dir = os.path.join(module_folder, subdir)
file_list.extend(getFileList(data_dir))
if file_list == []:
msg = "No files or folders found for '%s' in subfolder(s) '%s'." % (
module.getFullName(),
str(subdirs),
)
warning(msg)
yield ()
for f in file_list:
if "__pycache__" in f or f.endswith(".pyc"): # probably makes no sense
continue
target = f[filename_start:]
if folders_only is False:
item_set.add((f, target))
else:
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def _getQmlFileList(self, dlls):
qml_plugin_dir = self._getQmlDirectory()
# List all file types of the QML plugin folder that are datafiles and not DLLs.
datafile_suffixes = (
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
".mesh",
".frag",
)
if dlls:
ignore_suffixes = datafile_suffixes
only_suffixes = ()
else:
ignore_suffixes = ()
only_suffixes = datafile_suffixes
return getFileList(
qml_plugin_dir,
ignore_filenames=("qmldir",),
ignore_suffixes=ignore_suffixes,
only_suffixes=only_suffixes,
)
def copyQmlFiles(self, full_name, target_plugin_dir):
for plugin_dir in self.getQtPluginDirs():
qml_plugin_dir = os.path.normpath(
os.path.join(plugin_dir, "..", "qml"))
if os.path.exists(qml_plugin_dir):
break
else:
self.sysexit("Error, no such Qt plugin family: qml")
qml_target_dir = os.path.normpath(
self._getQmlTargetDir(target_plugin_dir))
self.info("Copying Qt plug-ins 'qml' to '%s'." % (qml_target_dir))
copyTree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
return [(
filename,
os.path.join(qml_target_dir,
os.path.relpath(filename, qml_plugin_dir)),
full_name,
) for filename in getFileList(qml_plugin_dir)
if not filename.endswith((
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)) if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"]
def setupCacheHashSalt(test_code_path):
assert os.path.exists(test_code_path)
if os.path.exists(os.path.join(test_code_path, ".git")):
git_cmd = ["git", "ls-tree", "-r", "HEAD", test_code_path]
process = subprocess.Popen(
args=git_cmd,
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout_git, stderr_git = process.communicate()
assert process.returncode == 0, stderr_git
salt_value = hashlib.md5(stdout_git)
else:
salt_value = hashlib.md5()
for filename in getFileList(test_code_path):
if filename.endswith(".py"):
salt_value.update(getFileContents(filename, mode="rb"))
os.environ["NUITKA_HASH_SALT"] = salt_value.hexdigest()
def considerExtraDlls(self, dist_dir, module):
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = os.path.join(
dist_dir,
full_name,
"qt-plugins"
)
shutil.copytree(
plugin_dir,
target_plugin_dir
)
plugin_options = self.getPluginOptions()
if not plugin_options:
plugin_options.append("all")
if plugin_options == ["sensible"]:
plugin_options = ["imageformats", "iconengines", "mediaservice", "printsupport"]
info(
"Copying '%s' Qt plug-ins to '%s'." % (
','.join(plugin_options),
target_plugin_dir
)
)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors = False)
for plugin_candidate in plugin_options:
if not os.path.isdir(os.path.join(target_plugin_dir, plugin_candidate)):
sys.exit("Error, no such Qt plugin family: %s" % plugin_candidate)
return [
(
filename,
os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir)),
full_name
)
for filename in
getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir)))
]
return ()
def getExtraDlls(self, module):
if module.getFullName() == "zmq" and self.dll_directory is not None:
for dll_filename in getFileList(self.dll_directory):
yield makeDllEntryPoint(
source_path=dll_filename,
dest_path=os.path.join("pyzmq.libs",
os.path.basename(dll_filename)),
package_name="zmq",
)
def getPackageFiles(module, packages, folders_only):
"""Yield all (!) filenames in given package(s).
Notes:
This should be required in rare occasions only. The one example I know
is 'dns' when used by package 'eventlet'. Eventlet imports dns modules
only to replace them with 'green' (i.e. non-blocking) counterparts.
Args:
module: module object
packages: package name(s) - str or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest), if folders_only is False,
else tuples (_createEmptyDirNone, dest).
"""
# TODO: Maybe use isinstance(basestring) for this
if not hasattr(packages, "__getitem__"): # so should be a string type
packages = (packages, )
file_list = []
item_set = OrderedSet()
file_dirs = []
for package in packages:
pkg_base, pkg_dir = get_package_paths(package) # read package folders
if pkg_dir:
filename_start = len(pkg_base) # position of package name in dir
# read out the filenames
pkg_files = getFileList(pkg_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", ))
file_dirs.append(pkg_dir)
for f in pkg_files:
file_list.append((filename_start, f)) # append to file list
if not file_list: # safeguard for unexpected cases
msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % (
module.getFullName(),
packages,
file_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for filename_start, f in file_list: # re-read the collected filenames
target = f[filename_start:] # make part of name
if folders_only is False: # normal case: indeed copy the files
item_set.add((f, target))
else: # just create the empty folder structure
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def copyMplDataFiles(self, dist_dir):
""" Write matplotlib data files ('mpl-data')."""
matplotlib_info = self._getMatplotlibInfo()
if not os.path.isdir(matplotlib_info.data_path):
self.sysexit(
"mpl-data missing, matplotlib installation appears to be broken"
)
# Copy data files to dist folder
for fullname in getFileList(matplotlib_info.data_path):
filename = os.path.relpath(fullname, matplotlib_info.data_path)
if filename.endswith("matplotlibrc"): # handle config separately
continue
target_filename = os.path.join(dist_dir, "matplotlib", "mpl-data",
filename)
makePath(os.path.dirname(
target_filename)) # create intermediate folders
shutil.copyfile(fullname, target_filename)
old_lines = (open(
matplotlib_info.matplotlibrc_filename).read().splitlines()
) # old config file lines
new_lines = [] # new config file lines
found = False # checks whether backend definition encountered
for line in old_lines:
line = line.rstrip()
# omit meaningless lines
if line.startswith(
"#") and matplotlib_info.matplotlib_version < "3":
continue
new_lines.append(line)
if line.startswith(("backend ", "backend:")):
# old config file has a backend definition
found = True
if not found and matplotlib_info.matplotlib_version < "3":
# Set the backend, so even if it was run time determined, we now enforce it.
new_lines.append("backend: %s" % matplotlib_info.backend)
matplotlibrc_filename = os.path.join(dist_dir, "matplotlib",
"mpl-data", "matplotlibrc")
putTextFileContents(filename=matplotlibrc_filename, contents=new_lines)
self.info("Copied 'matplotlib/mpl-data'.")
def checkReleaseDocumentation():
documents = [
entry for entry in getFileList(".")
if entry.endswith(".rst") and not entry.startswith("web" + os.path.sep)
]
for document in ("README.rst", "Developer_Manual.rst", "Changelog.rst"):
assert document in documents, documents
for document in documents:
checkRstLint(document)
def getSubDirectoryFiles(module, subdirs, folders_only):
"""Yield filenames in given subdirs of the module.
Notes:
All filenames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - str or None or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest) are yielded if folders_only is False,
else tuples (_createEmptyDirNone, dest) are yielded.
"""
module_folder = module.getCompileTimeDirectory()
elements = module.getFullName().split(".")
filename_start = module_folder.find(elements[0])
file_list = []
item_set = OrderedSet()
if subdirs is None:
data_dirs = [module_folder]
elif isinstance(subdirs, basestring):
data_dirs = [os.path.join(module_folder, subdirs)]
else:
data_dirs = [os.path.join(module_folder, subdir) for subdir in subdirs]
# Gather the full file list, probably makes no sense to include bytecode files
file_list = sum(
(getFileList(data_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", )) for data_dir in data_dirs),
[],
)
if not file_list:
msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % (
module.getFullName(),
subdirs,
data_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for f in file_list:
target = f[filename_start:]
if folders_only is False:
item_set.add((f, target))
else:
item_set.add((_createEmptyDirNone, target))
for f in item_set:
yield f
def packDistFolderToOnefileWindows(dist_dir):
general.warning("Onefile mode is experimental on '%s'." % getOS())
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
onefile_output_filename = getResultFullpath(onefile=True)
# First need to create the bootstrap binary for unpacking.
_runOnefileScons(quiet=not Options.isShowScons())
# Make sure to copy the resources from the created binary to the bootstrap binary, these
# are icons and version information.
copyResourcesFromFileToFile(
source_filename=getResultFullpath(onefile=False),
target_filename=onefile_output_filename,
resource_kinds=(RT_ICON, RT_GROUP_ICON, RT_VERSION),
)
# Now need to append to payload it, potentially compressing it.
compression_indicator, compressor = _pickCompressor()
with open(onefile_output_filename, "ab") as output_file:
# Seeking to end of file seems necessary on Python2 at least, maybe it's
# just that tell reports wrong value initially.
output_file.seek(0, 2)
start_pos = output_file.tell()
output_file.write(b"KA" + compression_indicator)
# Move the binary to start immediately to the start position
start_binary = getResultFullpath(onefile=False)
file_list = getFileList(dist_dir)
file_list.remove(start_binary)
file_list.insert(0, start_binary)
for filename_full in file_list:
filename_relative = os.path.relpath(filename_full, dist_dir)
filename_encoded = filename_relative.encode("utf-16le") + b"\0\0"
output_file.write(filename_encoded)
with open(filename_full, "rb") as input_file:
compressed = compressor(input_file.read())
output_file.write(struct.pack("Q", len(compressed)))
output_file.write(compressed)
# Using empty filename as a terminator.
output_file.write(b"\0\0")
output_file.write(struct.pack("Q", start_pos))
def addIncludedDataFilesFromPackageOptions():
"""Late data files, from plugins and user options that work with packages"""
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonPackage(
) or module.isUncompiledPythonPackage():
package_name = module.getFullName()
match, reason = package_name.matchesToShellPatterns(
patterns=getShallIncludePackageData())
if match:
package_directory = module.getCompileTimeDirectory()
pkg_filenames = getFileList(
package_directory,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
getSharedLibrarySuffixes(),
)
if pkg_filenames:
file_reason = "package '%s' %s" % (package_name, reason)
for pkg_filename in pkg_filenames:
rel_path = os.path.join(
package_name.asPath(),
os.path.relpath(pkg_filename, package_directory),
)
addIncludedDataFile(
makeIncludedDataFile(
source_path=pkg_filename,
dest_path=rel_path,
reason=file_reason,
tracer=options_logger,
tags="user,package_data",
))
# Circular dependency
from nuitka.plugins.Plugins import Plugins
# Plugins provide per module through this.
for module in ModuleRegistry.getDoneModules():
for included_datafile in Plugins.considerDataFiles(module=module):
addIncludedDataFile(included_datafile)
for included_datafile in getIncludedDataFiles():
if isinstance(included_datafile, (IncludedDataFile)):
Plugins.onDataFileTags(included_datafile, )
def findDLLs(self, full_name, target_plugin_dir):
# TODO: Change this to modern DLL entry points.
return [(
filename,
os.path.join(target_plugin_dir,
os.path.relpath(filename, plugin_dir)),
full_name,
) for plugin_dir in self.getQtPluginDirs()
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if not filename.endswith(".mesh") if os.path.exists(
os.path.join(target_plugin_dir,
os.path.relpath(filename, plugin_dir)))]
def packDistFolderToOnefileBootstrap(onefile_output_filename, dist_dir):
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while."
)
general.info("Running bootstrap binary compilation via Scons.")
# First need to create the bootstrap binary for unpacking.
_runOnefileScons(quiet=not Options.isShowScons())
executePostProcessingResources(manifest=None, onefile=True)
# Now need to append to payload it, potentially compressing it.
compression_indicator, compressor = _pickCompressor()
with open(onefile_output_filename, "ab") as output_file:
# Seeking to end of file seems necessary on Python2 at least, maybe it's
# just that tell reports wrong value initially.
output_file.seek(0, 2)
start_pos = output_file.tell()
output_file.write(b"KA" + compression_indicator)
# Move the binary to start immediately to the start position
start_binary = getResultFullpath(onefile=False)
file_list = getFileList(dist_dir, normalize=False)
file_list.remove(start_binary)
file_list.insert(0, start_binary)
for filename_full in file_list:
filename_relative = os.path.relpath(filename_full, dist_dir)
filename_encoded = filename_relative.encode("utf-16le") + b"\0\0"
output_file.write(filename_encoded)
with open(filename_full, "rb") as input_file:
compressed = compressor(input_file.read())
output_file.write(struct.pack("Q", len(compressed)))
output_file.write(compressed)
# Using empty filename as a terminator.
output_file.write(b"\0\0")
output_file.write(struct.pack("Q", start_pos))
def _getPackageFiles(module, *packages):
"""Yield all (!) filenames in given package(s).
Notes:
This should be required in rare occasions only. The one example I know
is 'dns' when used by package 'eventlet'. Eventlet imports dns modules
only to replace them with 'green' (i.e. non-blocking) counterparts.
Args:
module: module object
packages: package name(s) - str or tuple
Yields:
Tuples of paths (source, dest)
"""
file_list = []
item_set = OrderedSet()
file_dirs = []
for package in packages:
pkg_base, pkg_dir = get_package_paths(package) # read package folders
if pkg_dir:
filename_start = len(pkg_base) # position of package name in dir
# read out the filenames
pkg_files = getFileList(
pkg_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",)
)
file_dirs.append(pkg_dir)
for f in pkg_files:
file_list.append((filename_start, f)) # append to file list
if not file_list: # safeguard for unexpected cases
msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % (
module.getFullName(),
packages,
file_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
for filename_start, f in file_list: # re-read the collected filenames
target = f[filename_start:] # make part of name
item_set.add((f, target))
for f in item_set:
yield f
def _addIncludedDataFilesFromFileOptions():
for pattern, src, dest, arg in getShallIncludeDataFiles():
filenames = resolveShellPatternToFilenames(pattern)
if not filenames:
options_logger.warning(
"No matching data file to be included for '%s'." % pattern)
for filename in filenames:
file_reason = "specified data file '%s' on command line" % arg
if src is None:
rel_path = dest
if rel_path.endswith(("/", os.path.sep)):
rel_path = os.path.join(rel_path,
os.path.basename(filename))
else:
rel_path = os.path.join(dest, relpath(filename, src))
yield makeIncludedDataFile(filename,
rel_path,
file_reason,
tracer=options_logger,
tags="user")
for src, dest in getShallIncludeDataDirs():
filenames = getFileList(src)
if not filenames:
options_logger.warning("No files in directory '%s.'" % src)
for filename in filenames:
relative_filename = relpath(filename, src)
file_reason = "specified data dir %r on command line" % src
rel_path = os.path.join(dest, relative_filename)
yield makeIncludedDataFile(filename,
rel_path,
file_reason,
tracer=options_logger,
tags="user")
def considerExtraDlls(self, dist_dir, module):
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
shutil.copytree(plugin_dir, target_plugin_dir)
info("Copying all Qt plug-ins to '%s'." % target_plugin_dir)
return [(plugin_dir, filename, full_name)
for filename in getFileList(target_plugin_dir)
if not filename.endswith(".qml")]
return ()
def _findQtPluginDLLs(self):
for qt_plugins_dir in self.getQtPluginDirs():
for filename in getFileList(qt_plugins_dir):
filename_relative = os.path.relpath(filename,
start=qt_plugins_dir)
qt_plugin_name = filename_relative.split(os.path.sep, 1)[0]
if not self.hasQtPluginSelected(qt_plugin_name):
continue
yield self.makeDllEntryPoint(
source_path=filename,
dest_path=os.path.join(
self.binding_name,
"qt-plugins",
filename_relative,
),
package_name=self.binding_package_name,
)
def copyMplDataFiles(module, dist_dir):
""" Write matplotlib data files ('mpl-data')."""
data_dir = os.path.join(module.getCompileTimeDirectory(),
"mpl-data") # must exist
if not os.path.isdir(data_dir):
sys.exit("mpl-data missing: matplotlib installation is broken")
matplotlibrc, backend = getMatplotlibRc() # get matplotlibrc, backend
prefix = os.path.join("matplotlib", "mpl-data")
for item in getFileList(data_dir): # copy data files to dist folder
if item.endswith("matplotlibrc"): # handle config separately
continue
idx = item.find(
prefix) # need string starting with 'matplotlib/mpl-data'
tar_file = os.path.join(dist_dir, item[idx:])
makePath(os.path.dirname(tar_file)) # create intermediate folders
shutil.copyfile(item, tar_file)
old_lines = open(matplotlibrc).read().splitlines() # old config file lines
new_lines = ["# modified by Nuitka plugin 'numpy'"
] # new config file lines
found = False # checks whether backend definition encountered
for line in old_lines:
line = line.strip() # omit meaningless lines
if line.startswith("#") or line == "":
continue
new_lines.append(line)
if line.startswith(("backend ", "backend:")):
found = True # old config file has a backend definition
new_lines.append("# backend definition copied from installation")
if not found:
# get the string from interpreted mode and insert it in matplotlibrc
new_lines.append("backend: %s" % backend)
matplotlibrc = os.path.join(dist_dir, prefix, "matplotlibrc")
outfile = open(matplotlibrc, "w")
outfile.write("\n".join(new_lines))
outfile.close()
def _getWebviewFiles(module, dlls):
# TODO: Clarify non-Windows needs.
if not isWin32Windows():
return
webview_libdir = os.path.join(module.getCompileTimeDirectory(), "lib")
for filename in getFileList(webview_libdir):
filename_relative = os.path.relpath(filename, webview_libdir)
if getArchitecture() == "x86":
if "x64" in filename_relative:
continue
else:
if "x86" in filename_relative:
continue
is_dll = hasFilenameExtension(filename_relative, ".dll")
if dlls and not is_dll or not dlls and is_dll:
continue
yield filename, filename_relative
def _getSubDirectoryFolders(module, subdirs):
"""Get dirnames in given subdirs of the module.
Notes:
All dirnames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - tuple
Returns:
makeIncludedEmptyDirectories of found dirnames.
"""
module_dir = module.getCompileTimeDirectory()
file_list = []
data_dirs = [os.path.join(module_dir, subdir) for subdir in subdirs]
# Gather the full file list, probably makes no sense to include bytecode files
file_list = sum(
(
getFileList(
data_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",)
)
for data_dir in data_dirs
),
[],
)
if not file_list:
msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % (
module.getFullName(),
subdirs,
data_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
is_package = module.isCompiledPythonPackage() or module.isUncompiledPythonPackage()
# We need to preserve the package target path in the dist folder.
if is_package:
package_part = module.getFullName().asPath()
else:
package = module.getFullName().getPackageName()
if package is None:
package_part = ""
else:
package_part = package.asPath()
item_set = OrderedSet()
for f in file_list:
target = os.path.join(package_part, os.path.relpath(f, module_dir))
dir_name = os.path.dirname(target)
item_set.add(dir_name)
return makeIncludedEmptyDirectories(
source_path=module_dir,
dest_paths=item_set,
reason="Subdirectories of module %s" % module.getFullName(),
)
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
full_name = module.getFullName()
elements = full_name.split(".")
plugin_dirs = None
if elements[0] in ("PyQt4", "PyQt5"):
qt_version = int(elements[0][-1])
plugin_dirs = self.getPyQtPluginDirs(qt_version)
if full_name in ("PyQt4", "PyQt5"):
if not plugin_dirs:
sys.exit("Error, failed to detect %s plugin directories." % full_name)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = set(self.qt_plugins.split(","))
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
"platforms",
)
if self.hasPluginFamily(plugin_dirs, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
self.info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
for plugin_dir in plugin_dirs:
copyTree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for plugin_dir in plugin_dirs
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if isWin32Windows():
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_files = sum(
(
getFileList(qt_bin_dir)
for qt_bin_dir in self._getQtBinDirs(plugin_dirs)
),
[],
)
self.info("Copying OpenSSL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if "qml" in plugin_options or "all" in plugin_options:
for plugin_dir in plugin_dirs:
qml_plugin_dir = os.path.normpath(
os.path.join(plugin_dir, "..", "qml")
)
if os.path.exists(qml_plugin_dir):
break
else:
sys.exit("Error, no such Qt plugin family: qml")
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
self.info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
copyTree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
# Also copy required OpenGL DLLs on Windows
if isWin32Windows():
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
self.info("Copying OpenGL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
elif full_name == "PyQt5.QtNetwork":
if not isWin32Windows():
dll_path = locateDLL("crypto")
if dll_path is None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
dll_path = locateDLL("ssl")
if dll_path is not None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
elif (
full_name.startswith(("PyQt4.QtWebEngine", "PyQt5.QtWebEngine"))
and not self.webengine_done
):
self.webengine_done = True # prevent multiple copies
self.info("Copying QtWebEngine components")
plugin_parent = os.path.dirname(plugin_dirs[0])
if isWin32Windows():
bin_dir = os.path.join(plugin_parent, "bin")
else: # TODO verify this for non-Windows!
bin_dir = os.path.join(plugin_parent, "libexec")
target_bin_dir = os.path.join(dist_dir)
for f in os.listdir(bin_dir):
if f.startswith("QtWebEngineProcess"):
shutil.copy(os.path.join(bin_dir, f), target_bin_dir)
resources_dir = os.path.join(plugin_parent, "resources")
target_resources_dir = os.path.join(dist_dir)
for f in os.listdir(resources_dir):
shutil.copy(os.path.join(resources_dir, f), target_resources_dir)
translations_dir = os.path.join(plugin_parent, "translations")
pos = len(translations_dir) + 1
target_translations_dir = os.path.join(
dist_dir, elements[0], "Qt", "translations"
)
for f in getFileList(translations_dir):
tar_f = os.path.join(target_translations_dir, f[pos:])
makePath(os.path.dirname(tar_f))
shutil.copyfile(f, tar_f)
return ()
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = self.getPluginOptions()
plugin_options = set(plugin_options)
if not plugin_options:
plugin_options.add("sensible")
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
"platforms",
)
if self.hasPluginFamily(plugin_dir, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
shutil.copytree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if os.name == "nt":
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "bin"))
qt_bin_files = getFileList(qt_bin_dir)
info("Copying OpenSSL DLLs to %r" % (dist_dir,))
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if "qml" in plugin_options or "all" in plugin_options:
qml_plugin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "qml"))
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
shutil.copytree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
# Also copy required OpenGL DLLs on Windows
if os.name == "nt":
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
info("Copying OpenGL DLLs to %r" % (dist_dir,))
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
return ()
def considerDataFiles(self, module):
full_name = module.getFullName()
if full_name == self.binding_name and (
"qml" in self.getQtPluginsSelected()
or "all" in self.getQtPluginsSelected()):
qml_plugin_dir = self._getQmlDirectory()
qml_target_dir = self._getQmlTargetDir()
self.info("Including Qt plugins 'qml' below '%s'." %
qml_target_dir)
for filename in self._getQmlFileList(dlls=False):
filename_relative = os.path.relpath(filename, qml_plugin_dir)
yield self.makeIncludedDataFile(
source_path=filename,
dest_path=os.path.join(
qml_target_dir,
filename_relative,
),
reason="Qt QML datafile",
tags="qml",
)
elif self.isQtWebEngineModule(
full_name) and not self.webengine_done_data:
self.webengine_done_data = True
# TODO: This is probably wrong/not needed on macOS
if not isMacOS():
yield self.makeIncludedGeneratedDataFile(
data="""\
[Paths]
Prefix = .
""",
dest_path="qt6.conf"
if "6" in self.binding_name else "qt.conf",
reason="QtWebEngine needs Qt configuration file",
)
resources_dir = self._getResourcesPath()
for filename, filename_relative in listDir(resources_dir):
yield self.makeIncludedDataFile(
source_path=filename,
dest_path=os.path.join(self._getResourcesTargetDir(),
filename_relative),
reason="Qt resources",
)
if not self.no_qt_translations:
translations_path = self._getTranslationsPath()
for filename in getFileList(translations_path):
filename_relative = os.path.relpath(
filename, translations_path)
dest_path = self._getTranslationsTargetDir()
yield self.makeIncludedDataFile(
source_path=filename,
dest_path=os.path.join(dest_path, filename_relative),
reason="Qt translation",
tags="translation",
)
def getExtraDlls(self, module):
# pylint: disable=too-many-branches
full_name = module.getFullName()
if full_name == self.binding_name:
if not self.getQtPluginDirs():
self.sysexit(
"Error, failed to detect '%s' plugin directories." %
self.binding_name)
target_plugin_dir = os.path.join(full_name.asPath(), "qt-plugins")
self.info("Including Qt plugins '%s' below '%s'." % (
",".join(
sorted(
x for x in self.getQtPluginsSelected() if x != "xml")),
target_plugin_dir,
))
# TODO: Yielding a generator should become OK too.
for r in self._findQtPluginDLLs():
yield r
if isWin32Windows():
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_files = sum(
(getFileList(qt_bin_dir)
for qt_bin_dir in self._getQtBinDirs()),
[],
)
count = 0
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
yield self.makeDllEntryPoint(
source_path=filename,
dest_path=basename,
package_name=full_name,
)
count += 1
self.reportFileCount(full_name, count, section="OpenSSL")
if ("qml" in self.getQtPluginsSelected()
or "all" in self.getQtPluginsSelected()):
qml_plugin_dir = self._getQmlDirectory()
qml_target_dir = self._getQmlTargetDir()
for filename in self._getQmlFileList(dlls=True):
filename_relative = os.path.relpath(
filename, qml_plugin_dir)
yield self.makeDllEntryPoint(source_path=filename,
dest_path=os.path.join(
qml_target_dir,
filename_relative,
),
package_name=full_name
# reason="Qt QML plugin DLL",
)
# Also copy required OpenGL DLLs on Windows
if isWin32Windows():
opengl_dlls = ("libegl.dll", "libglesv2.dll",
"opengl32sw.dll")
count = 0
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"):
yield self.makeDllEntryPoint(
source_path=filename,
dest_path=basename,
package_name=full_name,
)
self.reportFileCount(full_name, count, section="OpenGL")
elif full_name == self.binding_name + ".QtNetwork":
if not isWin32Windows():
dll_path = self.locateDLL("crypto")
if dll_path is not None:
yield self.makeDllEntryPoint(
source_path=dll_path,
dest_path=os.path.basename(dll_path),
package_name=full_name,
)
dll_path = self.locateDLL("ssl")
if dll_path is not None:
yield self.makeDllEntryPoint(
source_path=dll_path,
dest_path=os.path.basename(dll_path),
package_name=full_name,
)
elif self.isQtWebEngineModule(
full_name) and not self.webengine_done_binaries:
self.webengine_done_binaries = True # prevent multiple copies
self.info("Including QtWebEngine executable.")
qt_web_engine_dir = self._getLibraryExecutablePath()
for filename, filename_relative in listDir(qt_web_engine_dir):
if filename_relative.startswith("QtWebEngineProcess"):
yield makeExeEntryPoint(
source_path=filename,
dest_path=os.path.join(self._getWebEngineTargetDir(),
filename_relative),
package_name=full_name,
)
break
else:
self.sysexit(
"Error, cannot locate QtWebEngineProcess executable at '%s'."
% qt_web_engine_dir)
def attachOnefilePayload(dist_dir, onefile_output_filename, start_binary,
expect_compression):
# Somewhat detail rich, pylint: disable=too-many-locals
compression_indicator, compressor = getCompressorFunction(
expect_compression=expect_compression)
with _openBinaryFileForAppending(onefile_output_filename) as output_file:
# Seeking to end of file seems necessary on Python2 at least, maybe it's
# just that tell reports wrong value initially.
output_file.seek(0, 2)
start_pos = output_file.tell()
output_file.write(b"KA" + compression_indicator)
# Move the binary to start immediately to the start position
file_list = getFileList(dist_dir, normalize=False)
file_list.remove(start_binary)
file_list.insert(0, start_binary)
if isWin32Windows():
filename_encoding = "utf-16le"
else:
filename_encoding = "utf8"
payload_size = 0
setupProgressBar(
stage="Onefile Payload",
unit="module",
total=len(file_list),
)
with compressor(output_file) as compressed_file:
for filename_full in file_list:
filename_relative = os.path.relpath(filename_full, dist_dir)
reportProgressBar(
item=filename_relative,
update=False,
)
filename_encoded = (filename_relative +
"\0").encode(filename_encoding)
compressed_file.write(filename_encoded)
payload_size += len(filename_encoded)
with open(filename_full, "rb") as input_file:
input_file.seek(0, 2)
input_size = input_file.tell()
input_file.seek(0, 0)
compressed_file.write(struct.pack("Q", input_size))
shutil.copyfileobj(input_file, compressed_file)
payload_size += input_size + 8
reportProgressBar(
item=filename_relative,
update=True,
)
# Using empty filename as a terminator.
filename_encoded = "\0".encode(filename_encoding)
compressed_file.write(filename_encoded)
payload_size += len(filename_encoded)
compressed_size = compressed_file.tell()
if compression_indicator == b"Y":
onefile_logger.info(
"Onefile payload compression ratio (%.2f%%) size %d to %d." % (
(float(compressed_size) / payload_size) * 100,
payload_size,
compressed_size,
))
if isWin32Windows():
# add padding to have the start position at a double world boundary
# this is needed on windows so that a possible certificate immediately
# follows the start position
pad = output_file.tell() % 8
if pad != 0:
output_file.write(bytes(8 - pad))
output_file.seek(0, 2)
end_pos = output_file.tell()
# Size of the payload data plus the size of that size storage, so C code can
# jump directly to it.
output_file.write(struct.pack("Q", end_pos - start_pos))
closeProgressBar()
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
full_name = module.getFullName()
if full_name == self.binding_name:
if not self.getQtPluginDirs():
self.sysexit(
"Error, failed to detect %r plugin directories." % self.binding_name
)
target_plugin_dir = os.path.join(dist_dir, full_name.asPath(), "qt-plugins")
self.info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(
sorted(x for x in self.getQtPluginsSelected() if x != "xml")
),
target_plugin_dir,
)
)
# TODO: Change this to filtering copyTree while it's doing it.
for plugin_dir in self.getQtPluginDirs():
copyTree(plugin_dir, target_plugin_dir)
if "all" not in self.getQtPluginsSelected():
for plugin_candidate in getSubDirectories(target_plugin_dir):
if (
os.path.basename(plugin_candidate)
not in self.getQtPluginsSelected()
):
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in self.getQtPluginsSelected():
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
self.sysexit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = self.findDLLs(
full_name=full_name,
target_plugin_dir=target_plugin_dir,
)
if isWin32Windows():
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_files = sum(
(getFileList(qt_bin_dir) for qt_bin_dir in self._getQtBinDirs()),
[],
)
self.info("Copying OpenSSL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if (
"qml" in self.getQtPluginsSelected()
or "all" in self.getQtPluginsSelected()
):
result += self.copyQmlFiles(
full_name=full_name,
target_plugin_dir=target_plugin_dir,
)
# Also copy required OpenGL DLLs on Windows
if isWin32Windows():
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
self.info("Copying OpenGL DLLs to %r." % dist_dir)
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
elif full_name == self.binding_name + ".QtNetwork":
if not isWin32Windows():
dll_path = locateDLL("crypto")
if dll_path is not None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
dll_path = locateDLL("ssl")
if dll_path is not None:
dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path))
shutil.copy(dll_path, dist_dll_path)
elif (
full_name
in (
self.binding_name + ".QtWebEngine",
self.binding_name + ".QtWebEngineCore",
self.binding_name + ".QtWebEngineWidgets",
)
and not self.webengine_done
):
self.webengine_done = True # prevent multiple copies
self.info("Copying QtWebEngine components")
plugin_parent = os.path.dirname(self.getQtPluginDirs()[0])
if isWin32Windows():
bin_dir = os.path.join(plugin_parent, "bin")
else: # TODO verify this for non-Windows!
bin_dir = os.path.join(plugin_parent, "libexec")
target_bin_dir = os.path.join(dist_dir)
for f in os.listdir(bin_dir):
if f.startswith("QtWebEngineProcess"):
shutil.copy(os.path.join(bin_dir, f), target_bin_dir)
resources_dir = os.path.join(plugin_parent, "resources")
target_resources_dir = os.path.join(dist_dir)
for f in os.listdir(resources_dir):
shutil.copy(os.path.join(resources_dir, f), target_resources_dir)
translations_dir = os.path.join(plugin_parent, "translations")
pos = len(translations_dir) + 1
target_translations_dir = os.path.join(
dist_dir,
full_name.getTopLevelPackageName().asPath(),
"Qt",
"translations",
)
for f in getFileList(translations_dir):
tar_f = os.path.join(target_translations_dir, f[pos:])
makePath(os.path.dirname(tar_f))
shutil.copyfile(f, tar_f)
return ()
def considerExtraDlls(self, dist_dir, module):
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dir, = self.getPyQtPluginDirs(qt_version)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = self.getPluginOptions()
plugin_options = set(plugin_options)
if not plugin_options:
plugin_options.add("sensible")
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
)
if self.hasPluginFamily(plugin_dir, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
# Seems platforms is required on Windows.
if os.name == "nt":
plugin_options.add("platforms")
info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
shutil.copytree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if "qml" in plugin_options or "all" in plugin_options:
qml_plugin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "qml"))
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
shutil.copytree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
return result
return ()
def considerExtraDlls(self, dist_dir, module):
# pylint: disable=too-many-branches,too-many-locals
full_name = module.getFullName()
if full_name in ("PyQt4", "PyQt5"):
qt_version = int(full_name[-1])
plugin_dirs = self.getPyQtPluginDirs(qt_version)
if not plugin_dirs:
sys.exit("Error, failed to detect %s plugin directories." % full_name)
target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins")
plugin_options = self.getPluginOptions()
plugin_options = set(plugin_options)
# Default to using sensible plugins.
if not plugin_options:
plugin_options.add("sensible")
if "sensible" in plugin_options:
# Most used ones with low dependencies.
plugin_options.update(
tuple(
family
for family in (
"imageformats",
"iconengines",
"mediaservice",
"printsupport",
"platforms",
)
if self.hasPluginFamily(plugin_dirs, family)
)
)
plugin_options.remove("sensible")
# Make sure the above didn't detect nothing, which would be
# indicating the check to be bad.
assert plugin_options
info(
"Copying Qt plug-ins '%s' to '%s'."
% (
",".join(sorted(x for x in plugin_options if x != "xml")),
target_plugin_dir,
)
)
for plugin_dir in plugin_dirs:
copyTree(plugin_dir, target_plugin_dir)
if "all" not in plugin_options:
for plugin_candidate in getSubDirectories(target_plugin_dir):
if os.path.basename(plugin_candidate) not in plugin_options:
removeDirectory(plugin_candidate, ignore_errors=False)
for plugin_candidate in plugin_options:
if plugin_candidate == "qml":
continue
if not os.path.isdir(
os.path.join(target_plugin_dir, plugin_candidate)
):
sys.exit(
"Error, no such Qt plugin family: %s" % plugin_candidate
)
result = [
(
filename,
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
),
full_name,
)
for plugin_dir in plugin_dirs
for filename in getFileList(plugin_dir)
if not filename.endswith(".qml")
if os.path.exists(
os.path.join(
target_plugin_dir, os.path.relpath(filename, plugin_dir)
)
)
]
if isWin32Windows():
# Those 2 vars will be used later, just saving some resources
# by caching the files list
qt_bin_files = sum(
(
getFileList(qt_bin_dir)
for qt_bin_dir in self._getQtBinDirs(plugin_dirs)
),
[],
)
info("Copying OpenSSL DLLs to %r" % (dist_dir,))
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in ("libeay32.dll", "ssleay32.dll"):
shutil.copy(filename, os.path.join(dist_dir, basename))
if "qml" in plugin_options or "all" in plugin_options:
for plugin_dir in plugin_dirs:
qml_plugin_dir = os.path.normpath(
os.path.join(plugin_dir, "..", "qml")
)
if os.path.exists(qml_plugin_dir):
break
else:
sys.exit("Error, no such Qt plugin family: qml")
qml_target_dir = os.path.normpath(
os.path.join(target_plugin_dir, "..", "Qt", "qml")
)
info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir))
copyTree(qml_plugin_dir, qml_target_dir)
# We try to filter here, not for DLLs.
result += [
(
filename,
os.path.join(
qml_target_dir, os.path.relpath(filename, qml_plugin_dir)
),
full_name,
)
for filename in getFileList(qml_plugin_dir)
if not filename.endswith(
(
".qml",
".qmlc",
".qmltypes",
".js",
".jsc",
".png",
".ttf",
".metainfo",
)
)
if not os.path.isdir(filename)
if not os.path.basename(filename) == "qmldir"
]
# Also copy required OpenGL DLLs on Windows
if isWin32Windows():
opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll")
info("Copying OpenGL DLLs to %r" % (dist_dir,))
for filename in qt_bin_files:
basename = os.path.basename(filename).lower()
if basename in opengl_dlls or basename.startswith(
"d3dcompiler_"
):
shutil.copy(filename, os.path.join(dist_dir, basename))
return result
return ()
def copyDataFiles(dist_dir):
"""Copy the data files needed for standalone distribution.
Args:
dist_dir: The distribution folder under creation
Notes:
This is for data files only, not DLLs or even extension modules,
those must be registered as entry points, and would not go through
necessary handling if provided like this.
"""
# Many details to deal with, pylint: disable=too-many-branches,too-many-locals
for pattern, dest, arg in Options.getShallIncludeDataFiles():
filenames = resolveShellPatternToFilenames(pattern)
if not filenames:
inclusion_logger.warning("No match data file to be included: %r" %
pattern)
for filename in filenames:
file_reason = "specified data file %r on command line" % arg
rel_path = dest
if rel_path.endswith(("/", os.path.sep)):
rel_path = os.path.join(rel_path, os.path.basename(filename))
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
for src, dest in Options.getShallIncludeDataDirs():
filenames = getFileList(src)
if not filenames:
inclusion_logger.warning("No files in directory" % src)
for filename in filenames:
relative_filename = relpath(filename, src)
file_reason = "specified data dir %r on command line" % src
rel_path = os.path.join(dest, relative_filename)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(filename, rel_path, file_reason),
)
# Cyclic dependency
from nuitka import ModuleRegistry
for module in ModuleRegistry.getDoneModules():
for plugin, included_datafile in Plugins.considerDataFiles(module):
_handleDataFile(dist_dir=dist_dir,
tracer=plugin,
included_datafile=included_datafile)
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonPackage(
) or module.isUncompiledPythonPackage():
package_name = module.getFullName()
match, reason = package_name.matchesToShellPatterns(
patterns=Options.getShallIncludePackageData())
if match:
package_directory = module.getCompileTimeDirectory()
pkg_filenames = getFileList(
package_directory,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") +
getSharedLibrarySuffixes(),
)
if pkg_filenames:
file_reason = "package '%s' %s" % (package_name, reason)
for pkg_filename in pkg_filenames:
rel_path = os.path.join(
package_name.asPath(),
os.path.relpath(pkg_filename, package_directory),
)
_handleDataFile(
dist_dir,
inclusion_logger,
makeIncludedDataFile(pkg_filename, rel_path,
file_reason),
)
def _getSubDirectoryFiles2(module, subdirs, folders_only):
"""Get filenames or dirnames in given subdirs of the module.
Notes:
All filenames in folders below one of the subdirs are recursively
retrieved and returned shortened to begin with the string of subdir.
Args:
module: module object
subdirs: sub folder name(s) - str or None or tuple
folders_only: (bool) indicate, whether just the folder structure should
be generated. In that case, an empty file named DUMMY will be
placed in each of these folders.
Yields:
Tuples of paths (source, dest) are yielded if folders_only is False,
else IncludedDataFile instances for empty dirs.
"""
module_dir = module.getCompileTimeDirectory()
file_list = []
if subdirs is None:
data_dirs = [module_dir]
elif isinstance(subdirs, basestring):
data_dirs = [os.path.join(module_dir, subdirs)]
else:
data_dirs = [os.path.join(module_dir, subdir) for subdir in subdirs]
# Gather the full file list, probably makes no sense to include bytecode files
file_list = sum(
(getFileList(data_dir,
ignore_dirs=("__pycache__", ),
ignore_suffixes=(".pyc", )) for data_dir in data_dirs),
[],
)
if not file_list:
msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % (
module.getFullName(),
subdirs,
data_dirs,
)
NuitkaPluginDataFileCollector.warning(msg)
is_package = module.isCompiledPythonPackage(
) or module.isUncompiledPythonPackage()
# We need to preserve the package target path in the dist folder.
if is_package:
package_part = module.getFullName().asPath()
else:
package = module.getFullName().getPackageName()
if package is None:
package_part = ""
else:
package_part = package.asPath()
item_set = OrderedSet()
for f in file_list:
target = os.path.join(package_part, os.path.relpath(f, module_dir))
if folders_only:
dir_name = os.path.dirname(target)
item_set.add(dir_name)
else:
item_set.add((f, target))
if folders_only:
return makeIncludedEmptyDirectories(
source_path=module_dir,
dest_paths=item_set,
reason="Subdirectories of module %s" % module.getFullName(),
)
return item_set