def _get_subdir_files(module, subdirs, folders_only): """Yield filenames in given subdirs of the module. Notes: All filenames in folders below one of the subdirs are recursively retrieved and returned shortened to begin with the string of subdir. Args: module: module object subdirs: sub folder name(s) - str or None or tuple folders_only: (bool) indicate, whether just the folder structure should be generated. In that case, an empty file named DUMMY will be placed in each of these folders. Yields: Tuples of paths (source, dest) are yielded if folders_only is False, else tuples (_createEmptyDirNone, dest) are yielded. """ module_folder = module.getCompileTimeDirectory() elements = module.getFullName().split(".") filename_start = module_folder.find(elements[0]) file_list = [] item_set = OrderedSet() if subdirs is None: file_list = getFileList(module_folder) elif type(subdirs) is str: data_dir = os.path.join(module_folder, subdirs) file_list = getFileList(data_dir) else: for subdir in subdirs: data_dir = os.path.join(module_folder, subdir) file_list.extend(getFileList(data_dir)) if file_list == []: msg = "No files or folders found for '%s' in subfolder(s) '%s'." % ( module.getFullName(), str(subdirs), ) warning(msg) yield () for f in file_list: if "__pycache__" in f or f.endswith(".pyc"): # probably makes no sense continue target = f[filename_start:] if folders_only is False: item_set.add((f, target)) else: item_set.add((_createEmptyDirNone, target)) for f in item_set: yield f
def _getQmlFileList(self, dlls): qml_plugin_dir = self._getQmlDirectory() # List all file types of the QML plugin folder that are datafiles and not DLLs. datafile_suffixes = ( ".qml", ".qmlc", ".qmltypes", ".js", ".jsc", ".png", ".ttf", ".metainfo", ".mesh", ".frag", ) if dlls: ignore_suffixes = datafile_suffixes only_suffixes = () else: ignore_suffixes = () only_suffixes = datafile_suffixes return getFileList( qml_plugin_dir, ignore_filenames=("qmldir",), ignore_suffixes=ignore_suffixes, only_suffixes=only_suffixes, )
def copyQmlFiles(self, full_name, target_plugin_dir): for plugin_dir in self.getQtPluginDirs(): qml_plugin_dir = os.path.normpath( os.path.join(plugin_dir, "..", "qml")) if os.path.exists(qml_plugin_dir): break else: self.sysexit("Error, no such Qt plugin family: qml") qml_target_dir = os.path.normpath( self._getQmlTargetDir(target_plugin_dir)) self.info("Copying Qt plug-ins 'qml' to '%s'." % (qml_target_dir)) copyTree(qml_plugin_dir, qml_target_dir) # We try to filter here, not for DLLs. return [( filename, os.path.join(qml_target_dir, os.path.relpath(filename, qml_plugin_dir)), full_name, ) for filename in getFileList(qml_plugin_dir) if not filename.endswith(( ".qml", ".qmlc", ".qmltypes", ".js", ".jsc", ".png", ".ttf", ".metainfo", )) if not os.path.isdir(filename) if not os.path.basename(filename) == "qmldir"]
def setupCacheHashSalt(test_code_path): assert os.path.exists(test_code_path) if os.path.exists(os.path.join(test_code_path, ".git")): git_cmd = ["git", "ls-tree", "-r", "HEAD", test_code_path] process = subprocess.Popen( args=git_cmd, stdin=getNullInput(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) stdout_git, stderr_git = process.communicate() assert process.returncode == 0, stderr_git salt_value = hashlib.md5(stdout_git) else: salt_value = hashlib.md5() for filename in getFileList(test_code_path): if filename.endswith(".py"): salt_value.update(getFileContents(filename, mode="rb")) os.environ["NUITKA_HASH_SALT"] = salt_value.hexdigest()
def considerExtraDlls(self, dist_dir, module): full_name = module.getFullName() if full_name in ("PyQt4", "PyQt5"): qt_version = int(full_name[-1]) plugin_dir, = self.getPyQtPluginDirs(qt_version) target_plugin_dir = os.path.join( dist_dir, full_name, "qt-plugins" ) shutil.copytree( plugin_dir, target_plugin_dir ) plugin_options = self.getPluginOptions() if not plugin_options: plugin_options.append("all") if plugin_options == ["sensible"]: plugin_options = ["imageformats", "iconengines", "mediaservice", "printsupport"] info( "Copying '%s' Qt plug-ins to '%s'." % ( ','.join(plugin_options), target_plugin_dir ) ) if "all" not in plugin_options: for plugin_candidate in getSubDirectories(target_plugin_dir): if os.path.basename(plugin_candidate) not in plugin_options: removeDirectory(plugin_candidate, ignore_errors = False) for plugin_candidate in plugin_options: if not os.path.isdir(os.path.join(target_plugin_dir, plugin_candidate)): sys.exit("Error, no such Qt plugin family: %s" % plugin_candidate) return [ ( filename, os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir)), full_name ) for filename in getFileList(plugin_dir) if not filename.endswith(".qml") if os.path.exists(os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir))) ] return ()
def getExtraDlls(self, module): if module.getFullName() == "zmq" and self.dll_directory is not None: for dll_filename in getFileList(self.dll_directory): yield makeDllEntryPoint( source_path=dll_filename, dest_path=os.path.join("pyzmq.libs", os.path.basename(dll_filename)), package_name="zmq", )
def getPackageFiles(module, packages, folders_only): """Yield all (!) filenames in given package(s). Notes: This should be required in rare occasions only. The one example I know is 'dns' when used by package 'eventlet'. Eventlet imports dns modules only to replace them with 'green' (i.e. non-blocking) counterparts. Args: module: module object packages: package name(s) - str or tuple folders_only: (bool) indicate, whether just the folder structure should be generated. In that case, an empty file named DUMMY will be placed in each of these folders. Yields: Tuples of paths (source, dest), if folders_only is False, else tuples (_createEmptyDirNone, dest). """ # TODO: Maybe use isinstance(basestring) for this if not hasattr(packages, "__getitem__"): # so should be a string type packages = (packages, ) file_list = [] item_set = OrderedSet() file_dirs = [] for package in packages: pkg_base, pkg_dir = get_package_paths(package) # read package folders if pkg_dir: filename_start = len(pkg_base) # position of package name in dir # read out the filenames pkg_files = getFileList(pkg_dir, ignore_dirs=("__pycache__", ), ignore_suffixes=(".pyc", )) file_dirs.append(pkg_dir) for f in pkg_files: file_list.append((filename_start, f)) # append to file list if not file_list: # safeguard for unexpected cases msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % ( module.getFullName(), packages, file_dirs, ) NuitkaPluginDataFileCollector.warning(msg) for filename_start, f in file_list: # re-read the collected filenames target = f[filename_start:] # make part of name if folders_only is False: # normal case: indeed copy the files item_set.add((f, target)) else: # just create the empty folder structure item_set.add((_createEmptyDirNone, target)) for f in item_set: yield f
def copyMplDataFiles(self, dist_dir): """ Write matplotlib data files ('mpl-data').""" matplotlib_info = self._getMatplotlibInfo() if not os.path.isdir(matplotlib_info.data_path): self.sysexit( "mpl-data missing, matplotlib installation appears to be broken" ) # Copy data files to dist folder for fullname in getFileList(matplotlib_info.data_path): filename = os.path.relpath(fullname, matplotlib_info.data_path) if filename.endswith("matplotlibrc"): # handle config separately continue target_filename = os.path.join(dist_dir, "matplotlib", "mpl-data", filename) makePath(os.path.dirname( target_filename)) # create intermediate folders shutil.copyfile(fullname, target_filename) old_lines = (open( matplotlib_info.matplotlibrc_filename).read().splitlines() ) # old config file lines new_lines = [] # new config file lines found = False # checks whether backend definition encountered for line in old_lines: line = line.rstrip() # omit meaningless lines if line.startswith( "#") and matplotlib_info.matplotlib_version < "3": continue new_lines.append(line) if line.startswith(("backend ", "backend:")): # old config file has a backend definition found = True if not found and matplotlib_info.matplotlib_version < "3": # Set the backend, so even if it was run time determined, we now enforce it. new_lines.append("backend: %s" % matplotlib_info.backend) matplotlibrc_filename = os.path.join(dist_dir, "matplotlib", "mpl-data", "matplotlibrc") putTextFileContents(filename=matplotlibrc_filename, contents=new_lines) self.info("Copied 'matplotlib/mpl-data'.")
def checkReleaseDocumentation(): documents = [ entry for entry in getFileList(".") if entry.endswith(".rst") and not entry.startswith("web" + os.path.sep) ] for document in ("README.rst", "Developer_Manual.rst", "Changelog.rst"): assert document in documents, documents for document in documents: checkRstLint(document)
def getSubDirectoryFiles(module, subdirs, folders_only): """Yield filenames in given subdirs of the module. Notes: All filenames in folders below one of the subdirs are recursively retrieved and returned shortened to begin with the string of subdir. Args: module: module object subdirs: sub folder name(s) - str or None or tuple folders_only: (bool) indicate, whether just the folder structure should be generated. In that case, an empty file named DUMMY will be placed in each of these folders. Yields: Tuples of paths (source, dest) are yielded if folders_only is False, else tuples (_createEmptyDirNone, dest) are yielded. """ module_folder = module.getCompileTimeDirectory() elements = module.getFullName().split(".") filename_start = module_folder.find(elements[0]) file_list = [] item_set = OrderedSet() if subdirs is None: data_dirs = [module_folder] elif isinstance(subdirs, basestring): data_dirs = [os.path.join(module_folder, subdirs)] else: data_dirs = [os.path.join(module_folder, subdir) for subdir in subdirs] # Gather the full file list, probably makes no sense to include bytecode files file_list = sum( (getFileList(data_dir, ignore_dirs=("__pycache__", ), ignore_suffixes=(".pyc", )) for data_dir in data_dirs), [], ) if not file_list: msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % ( module.getFullName(), subdirs, data_dirs, ) NuitkaPluginDataFileCollector.warning(msg) for f in file_list: target = f[filename_start:] if folders_only is False: item_set.add((f, target)) else: item_set.add((_createEmptyDirNone, target)) for f in item_set: yield f
def packDistFolderToOnefileWindows(dist_dir): general.warning("Onefile mode is experimental on '%s'." % getOS()) postprocessing_logger.info( "Creating single file from dist folder, this may take a while.") onefile_output_filename = getResultFullpath(onefile=True) # First need to create the bootstrap binary for unpacking. _runOnefileScons(quiet=not Options.isShowScons()) # Make sure to copy the resources from the created binary to the bootstrap binary, these # are icons and version information. copyResourcesFromFileToFile( source_filename=getResultFullpath(onefile=False), target_filename=onefile_output_filename, resource_kinds=(RT_ICON, RT_GROUP_ICON, RT_VERSION), ) # Now need to append to payload it, potentially compressing it. compression_indicator, compressor = _pickCompressor() with open(onefile_output_filename, "ab") as output_file: # Seeking to end of file seems necessary on Python2 at least, maybe it's # just that tell reports wrong value initially. output_file.seek(0, 2) start_pos = output_file.tell() output_file.write(b"KA" + compression_indicator) # Move the binary to start immediately to the start position start_binary = getResultFullpath(onefile=False) file_list = getFileList(dist_dir) file_list.remove(start_binary) file_list.insert(0, start_binary) for filename_full in file_list: filename_relative = os.path.relpath(filename_full, dist_dir) filename_encoded = filename_relative.encode("utf-16le") + b"\0\0" output_file.write(filename_encoded) with open(filename_full, "rb") as input_file: compressed = compressor(input_file.read()) output_file.write(struct.pack("Q", len(compressed))) output_file.write(compressed) # Using empty filename as a terminator. output_file.write(b"\0\0") output_file.write(struct.pack("Q", start_pos))
def addIncludedDataFilesFromPackageOptions(): """Late data files, from plugins and user options that work with packages""" # Cyclic dependency from nuitka import ModuleRegistry for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonPackage( ) or module.isUncompiledPythonPackage(): package_name = module.getFullName() match, reason = package_name.matchesToShellPatterns( patterns=getShallIncludePackageData()) if match: package_directory = module.getCompileTimeDirectory() pkg_filenames = getFileList( package_directory, ignore_dirs=("__pycache__", ), ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") + getSharedLibrarySuffixes(), ) if pkg_filenames: file_reason = "package '%s' %s" % (package_name, reason) for pkg_filename in pkg_filenames: rel_path = os.path.join( package_name.asPath(), os.path.relpath(pkg_filename, package_directory), ) addIncludedDataFile( makeIncludedDataFile( source_path=pkg_filename, dest_path=rel_path, reason=file_reason, tracer=options_logger, tags="user,package_data", )) # Circular dependency from nuitka.plugins.Plugins import Plugins # Plugins provide per module through this. for module in ModuleRegistry.getDoneModules(): for included_datafile in Plugins.considerDataFiles(module=module): addIncludedDataFile(included_datafile) for included_datafile in getIncludedDataFiles(): if isinstance(included_datafile, (IncludedDataFile)): Plugins.onDataFileTags(included_datafile, )
def findDLLs(self, full_name, target_plugin_dir): # TODO: Change this to modern DLL entry points. return [( filename, os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir)), full_name, ) for plugin_dir in self.getQtPluginDirs() for filename in getFileList(plugin_dir) if not filename.endswith(".qml") if not filename.endswith(".mesh") if os.path.exists( os.path.join(target_plugin_dir, os.path.relpath(filename, plugin_dir)))]
def packDistFolderToOnefileBootstrap(onefile_output_filename, dist_dir): postprocessing_logger.info( "Creating single file from dist folder, this may take a while." ) general.info("Running bootstrap binary compilation via Scons.") # First need to create the bootstrap binary for unpacking. _runOnefileScons(quiet=not Options.isShowScons()) executePostProcessingResources(manifest=None, onefile=True) # Now need to append to payload it, potentially compressing it. compression_indicator, compressor = _pickCompressor() with open(onefile_output_filename, "ab") as output_file: # Seeking to end of file seems necessary on Python2 at least, maybe it's # just that tell reports wrong value initially. output_file.seek(0, 2) start_pos = output_file.tell() output_file.write(b"KA" + compression_indicator) # Move the binary to start immediately to the start position start_binary = getResultFullpath(onefile=False) file_list = getFileList(dist_dir, normalize=False) file_list.remove(start_binary) file_list.insert(0, start_binary) for filename_full in file_list: filename_relative = os.path.relpath(filename_full, dist_dir) filename_encoded = filename_relative.encode("utf-16le") + b"\0\0" output_file.write(filename_encoded) with open(filename_full, "rb") as input_file: compressed = compressor(input_file.read()) output_file.write(struct.pack("Q", len(compressed))) output_file.write(compressed) # Using empty filename as a terminator. output_file.write(b"\0\0") output_file.write(struct.pack("Q", start_pos))
def _getPackageFiles(module, *packages): """Yield all (!) filenames in given package(s). Notes: This should be required in rare occasions only. The one example I know is 'dns' when used by package 'eventlet'. Eventlet imports dns modules only to replace them with 'green' (i.e. non-blocking) counterparts. Args: module: module object packages: package name(s) - str or tuple Yields: Tuples of paths (source, dest) """ file_list = [] item_set = OrderedSet() file_dirs = [] for package in packages: pkg_base, pkg_dir = get_package_paths(package) # read package folders if pkg_dir: filename_start = len(pkg_base) # position of package name in dir # read out the filenames pkg_files = getFileList( pkg_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",) ) file_dirs.append(pkg_dir) for f in pkg_files: file_list.append((filename_start, f)) # append to file list if not file_list: # safeguard for unexpected cases msg = "No files or folders found for '%s' in packages(s) '%r' (%r)." % ( module.getFullName(), packages, file_dirs, ) NuitkaPluginDataFileCollector.warning(msg) for filename_start, f in file_list: # re-read the collected filenames target = f[filename_start:] # make part of name item_set.add((f, target)) for f in item_set: yield f
def _addIncludedDataFilesFromFileOptions(): for pattern, src, dest, arg in getShallIncludeDataFiles(): filenames = resolveShellPatternToFilenames(pattern) if not filenames: options_logger.warning( "No matching data file to be included for '%s'." % pattern) for filename in filenames: file_reason = "specified data file '%s' on command line" % arg if src is None: rel_path = dest if rel_path.endswith(("/", os.path.sep)): rel_path = os.path.join(rel_path, os.path.basename(filename)) else: rel_path = os.path.join(dest, relpath(filename, src)) yield makeIncludedDataFile(filename, rel_path, file_reason, tracer=options_logger, tags="user") for src, dest in getShallIncludeDataDirs(): filenames = getFileList(src) if not filenames: options_logger.warning("No files in directory '%s.'" % src) for filename in filenames: relative_filename = relpath(filename, src) file_reason = "specified data dir %r on command line" % src rel_path = os.path.join(dest, relative_filename) yield makeIncludedDataFile(filename, rel_path, file_reason, tracer=options_logger, tags="user")
def considerExtraDlls(self, dist_dir, module): full_name = module.getFullName() if full_name in ("PyQt4", "PyQt5"): qt_version = int(full_name[-1]) plugin_dir, = self.getPyQtPluginDirs(qt_version) target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins") shutil.copytree(plugin_dir, target_plugin_dir) info("Copying all Qt plug-ins to '%s'." % target_plugin_dir) return [(plugin_dir, filename, full_name) for filename in getFileList(target_plugin_dir) if not filename.endswith(".qml")] return ()
def _findQtPluginDLLs(self): for qt_plugins_dir in self.getQtPluginDirs(): for filename in getFileList(qt_plugins_dir): filename_relative = os.path.relpath(filename, start=qt_plugins_dir) qt_plugin_name = filename_relative.split(os.path.sep, 1)[0] if not self.hasQtPluginSelected(qt_plugin_name): continue yield self.makeDllEntryPoint( source_path=filename, dest_path=os.path.join( self.binding_name, "qt-plugins", filename_relative, ), package_name=self.binding_package_name, )
def copyMplDataFiles(module, dist_dir): """ Write matplotlib data files ('mpl-data').""" data_dir = os.path.join(module.getCompileTimeDirectory(), "mpl-data") # must exist if not os.path.isdir(data_dir): sys.exit("mpl-data missing: matplotlib installation is broken") matplotlibrc, backend = getMatplotlibRc() # get matplotlibrc, backend prefix = os.path.join("matplotlib", "mpl-data") for item in getFileList(data_dir): # copy data files to dist folder if item.endswith("matplotlibrc"): # handle config separately continue idx = item.find( prefix) # need string starting with 'matplotlib/mpl-data' tar_file = os.path.join(dist_dir, item[idx:]) makePath(os.path.dirname(tar_file)) # create intermediate folders shutil.copyfile(item, tar_file) old_lines = open(matplotlibrc).read().splitlines() # old config file lines new_lines = ["# modified by Nuitka plugin 'numpy'" ] # new config file lines found = False # checks whether backend definition encountered for line in old_lines: line = line.strip() # omit meaningless lines if line.startswith("#") or line == "": continue new_lines.append(line) if line.startswith(("backend ", "backend:")): found = True # old config file has a backend definition new_lines.append("# backend definition copied from installation") if not found: # get the string from interpreted mode and insert it in matplotlibrc new_lines.append("backend: %s" % backend) matplotlibrc = os.path.join(dist_dir, prefix, "matplotlibrc") outfile = open(matplotlibrc, "w") outfile.write("\n".join(new_lines)) outfile.close()
def _getWebviewFiles(module, dlls): # TODO: Clarify non-Windows needs. if not isWin32Windows(): return webview_libdir = os.path.join(module.getCompileTimeDirectory(), "lib") for filename in getFileList(webview_libdir): filename_relative = os.path.relpath(filename, webview_libdir) if getArchitecture() == "x86": if "x64" in filename_relative: continue else: if "x86" in filename_relative: continue is_dll = hasFilenameExtension(filename_relative, ".dll") if dlls and not is_dll or not dlls and is_dll: continue yield filename, filename_relative
def _getSubDirectoryFolders(module, subdirs): """Get dirnames in given subdirs of the module. Notes: All dirnames in folders below one of the subdirs are recursively retrieved and returned shortened to begin with the string of subdir. Args: module: module object subdirs: sub folder name(s) - tuple Returns: makeIncludedEmptyDirectories of found dirnames. """ module_dir = module.getCompileTimeDirectory() file_list = [] data_dirs = [os.path.join(module_dir, subdir) for subdir in subdirs] # Gather the full file list, probably makes no sense to include bytecode files file_list = sum( ( getFileList( data_dir, ignore_dirs=("__pycache__",), ignore_suffixes=(".pyc",) ) for data_dir in data_dirs ), [], ) if not file_list: msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % ( module.getFullName(), subdirs, data_dirs, ) NuitkaPluginDataFileCollector.warning(msg) is_package = module.isCompiledPythonPackage() or module.isUncompiledPythonPackage() # We need to preserve the package target path in the dist folder. if is_package: package_part = module.getFullName().asPath() else: package = module.getFullName().getPackageName() if package is None: package_part = "" else: package_part = package.asPath() item_set = OrderedSet() for f in file_list: target = os.path.join(package_part, os.path.relpath(f, module_dir)) dir_name = os.path.dirname(target) item_set.add(dir_name) return makeIncludedEmptyDirectories( source_path=module_dir, dest_paths=item_set, reason="Subdirectories of module %s" % module.getFullName(), )
def considerExtraDlls(self, dist_dir, module): # pylint: disable=too-many-branches,too-many-locals,too-many-statements full_name = module.getFullName() elements = full_name.split(".") plugin_dirs = None if elements[0] in ("PyQt4", "PyQt5"): qt_version = int(elements[0][-1]) plugin_dirs = self.getPyQtPluginDirs(qt_version) if full_name in ("PyQt4", "PyQt5"): if not plugin_dirs: sys.exit("Error, failed to detect %s plugin directories." % full_name) target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins") plugin_options = set(self.qt_plugins.split(",")) if "sensible" in plugin_options: # Most used ones with low dependencies. plugin_options.update( tuple( family for family in ( "imageformats", "iconengines", "mediaservice", "printsupport", "platforms", ) if self.hasPluginFamily(plugin_dirs, family) ) ) plugin_options.remove("sensible") # Make sure the above didn't detect nothing, which would be # indicating the check to be bad. assert plugin_options self.info( "Copying Qt plug-ins '%s' to '%s'." % ( ",".join(sorted(x for x in plugin_options if x != "xml")), target_plugin_dir, ) ) for plugin_dir in plugin_dirs: copyTree(plugin_dir, target_plugin_dir) if "all" not in plugin_options: for plugin_candidate in getSubDirectories(target_plugin_dir): if os.path.basename(plugin_candidate) not in plugin_options: removeDirectory(plugin_candidate, ignore_errors=False) for plugin_candidate in plugin_options: if plugin_candidate == "qml": continue if not os.path.isdir( os.path.join(target_plugin_dir, plugin_candidate) ): sys.exit( "Error, no such Qt plugin family: %s" % plugin_candidate ) result = [ ( filename, os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ), full_name, ) for plugin_dir in plugin_dirs for filename in getFileList(plugin_dir) if not filename.endswith(".qml") if os.path.exists( os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ) ) ] if isWin32Windows(): # Those 2 vars will be used later, just saving some resources # by caching the files list qt_bin_files = sum( ( getFileList(qt_bin_dir) for qt_bin_dir in self._getQtBinDirs(plugin_dirs) ), [], ) self.info("Copying OpenSSL DLLs to %r." % dist_dir) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in ("libeay32.dll", "ssleay32.dll"): shutil.copy(filename, os.path.join(dist_dir, basename)) if "qml" in plugin_options or "all" in plugin_options: for plugin_dir in plugin_dirs: qml_plugin_dir = os.path.normpath( os.path.join(plugin_dir, "..", "qml") ) if os.path.exists(qml_plugin_dir): break else: sys.exit("Error, no such Qt plugin family: qml") qml_target_dir = os.path.normpath( os.path.join(target_plugin_dir, "..", "Qt", "qml") ) self.info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir)) copyTree(qml_plugin_dir, qml_target_dir) # We try to filter here, not for DLLs. result += [ ( filename, os.path.join( qml_target_dir, os.path.relpath(filename, qml_plugin_dir) ), full_name, ) for filename in getFileList(qml_plugin_dir) if not filename.endswith( ( ".qml", ".qmlc", ".qmltypes", ".js", ".jsc", ".png", ".ttf", ".metainfo", ) ) if not os.path.isdir(filename) if not os.path.basename(filename) == "qmldir" ] # Also copy required OpenGL DLLs on Windows if isWin32Windows(): opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll") self.info("Copying OpenGL DLLs to %r." % dist_dir) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in opengl_dlls or basename.startswith( "d3dcompiler_" ): shutil.copy(filename, os.path.join(dist_dir, basename)) return result elif full_name == "PyQt5.QtNetwork": if not isWin32Windows(): dll_path = locateDLL("crypto") if dll_path is None: dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path)) shutil.copy(dll_path, dist_dll_path) dll_path = locateDLL("ssl") if dll_path is not None: dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path)) shutil.copy(dll_path, dist_dll_path) elif ( full_name.startswith(("PyQt4.QtWebEngine", "PyQt5.QtWebEngine")) and not self.webengine_done ): self.webengine_done = True # prevent multiple copies self.info("Copying QtWebEngine components") plugin_parent = os.path.dirname(plugin_dirs[0]) if isWin32Windows(): bin_dir = os.path.join(plugin_parent, "bin") else: # TODO verify this for non-Windows! bin_dir = os.path.join(plugin_parent, "libexec") target_bin_dir = os.path.join(dist_dir) for f in os.listdir(bin_dir): if f.startswith("QtWebEngineProcess"): shutil.copy(os.path.join(bin_dir, f), target_bin_dir) resources_dir = os.path.join(plugin_parent, "resources") target_resources_dir = os.path.join(dist_dir) for f in os.listdir(resources_dir): shutil.copy(os.path.join(resources_dir, f), target_resources_dir) translations_dir = os.path.join(plugin_parent, "translations") pos = len(translations_dir) + 1 target_translations_dir = os.path.join( dist_dir, elements[0], "Qt", "translations" ) for f in getFileList(translations_dir): tar_f = os.path.join(target_translations_dir, f[pos:]) makePath(os.path.dirname(tar_f)) shutil.copyfile(f, tar_f) return ()
def considerExtraDlls(self, dist_dir, module): # pylint: disable=too-many-branches,too-many-locals full_name = module.getFullName() if full_name in ("PyQt4", "PyQt5"): qt_version = int(full_name[-1]) plugin_dir, = self.getPyQtPluginDirs(qt_version) target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins") plugin_options = self.getPluginOptions() plugin_options = set(plugin_options) if not plugin_options: plugin_options.add("sensible") if "sensible" in plugin_options: # Most used ones with low dependencies. plugin_options.update( tuple( family for family in ( "imageformats", "iconengines", "mediaservice", "printsupport", "platforms", ) if self.hasPluginFamily(plugin_dir, family) ) ) plugin_options.remove("sensible") # Make sure the above didn't detect nothing, which would be # indicating the check to be bad. assert plugin_options info( "Copying Qt plug-ins '%s' to '%s'." % ( ",".join(sorted(x for x in plugin_options if x != "xml")), target_plugin_dir, ) ) shutil.copytree(plugin_dir, target_plugin_dir) if "all" not in plugin_options: for plugin_candidate in getSubDirectories(target_plugin_dir): if os.path.basename(plugin_candidate) not in plugin_options: removeDirectory(plugin_candidate, ignore_errors=False) for plugin_candidate in plugin_options: if plugin_candidate == "qml": continue if not os.path.isdir( os.path.join(target_plugin_dir, plugin_candidate) ): sys.exit( "Error, no such Qt plugin family: %s" % plugin_candidate ) result = [ ( filename, os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ), full_name, ) for filename in getFileList(plugin_dir) if not filename.endswith(".qml") if os.path.exists( os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ) ) ] if os.name == "nt": # Those 2 vars will be used later, just saving some resources # by caching the files list qt_bin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "bin")) qt_bin_files = getFileList(qt_bin_dir) info("Copying OpenSSL DLLs to %r" % (dist_dir,)) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in ("libeay32.dll", "ssleay32.dll"): shutil.copy(filename, os.path.join(dist_dir, basename)) if "qml" in plugin_options or "all" in plugin_options: qml_plugin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "qml")) qml_target_dir = os.path.normpath( os.path.join(target_plugin_dir, "..", "Qt", "qml") ) info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir)) shutil.copytree(qml_plugin_dir, qml_target_dir) # We try to filter here, not for DLLs. result += [ ( filename, os.path.join( qml_target_dir, os.path.relpath(filename, qml_plugin_dir) ), full_name, ) for filename in getFileList(qml_plugin_dir) if not filename.endswith( ( ".qml", ".qmlc", ".qmltypes", ".js", ".jsc", ".png", ".ttf", ".metainfo", ) ) if not os.path.isdir(filename) if not os.path.basename(filename) == "qmldir" ] # Also copy required OpenGL DLLs on Windows if os.name == "nt": opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll") info("Copying OpenGL DLLs to %r" % (dist_dir,)) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in opengl_dlls or basename.startswith( "d3dcompiler_" ): shutil.copy(filename, os.path.join(dist_dir, basename)) return result return ()
def considerDataFiles(self, module): full_name = module.getFullName() if full_name == self.binding_name and ( "qml" in self.getQtPluginsSelected() or "all" in self.getQtPluginsSelected()): qml_plugin_dir = self._getQmlDirectory() qml_target_dir = self._getQmlTargetDir() self.info("Including Qt plugins 'qml' below '%s'." % qml_target_dir) for filename in self._getQmlFileList(dlls=False): filename_relative = os.path.relpath(filename, qml_plugin_dir) yield self.makeIncludedDataFile( source_path=filename, dest_path=os.path.join( qml_target_dir, filename_relative, ), reason="Qt QML datafile", tags="qml", ) elif self.isQtWebEngineModule( full_name) and not self.webengine_done_data: self.webengine_done_data = True # TODO: This is probably wrong/not needed on macOS if not isMacOS(): yield self.makeIncludedGeneratedDataFile( data="""\ [Paths] Prefix = . """, dest_path="qt6.conf" if "6" in self.binding_name else "qt.conf", reason="QtWebEngine needs Qt configuration file", ) resources_dir = self._getResourcesPath() for filename, filename_relative in listDir(resources_dir): yield self.makeIncludedDataFile( source_path=filename, dest_path=os.path.join(self._getResourcesTargetDir(), filename_relative), reason="Qt resources", ) if not self.no_qt_translations: translations_path = self._getTranslationsPath() for filename in getFileList(translations_path): filename_relative = os.path.relpath( filename, translations_path) dest_path = self._getTranslationsTargetDir() yield self.makeIncludedDataFile( source_path=filename, dest_path=os.path.join(dest_path, filename_relative), reason="Qt translation", tags="translation", )
def getExtraDlls(self, module): # pylint: disable=too-many-branches full_name = module.getFullName() if full_name == self.binding_name: if not self.getQtPluginDirs(): self.sysexit( "Error, failed to detect '%s' plugin directories." % self.binding_name) target_plugin_dir = os.path.join(full_name.asPath(), "qt-plugins") self.info("Including Qt plugins '%s' below '%s'." % ( ",".join( sorted( x for x in self.getQtPluginsSelected() if x != "xml")), target_plugin_dir, )) # TODO: Yielding a generator should become OK too. for r in self._findQtPluginDLLs(): yield r if isWin32Windows(): # Those 2 vars will be used later, just saving some resources # by caching the files list qt_bin_files = sum( (getFileList(qt_bin_dir) for qt_bin_dir in self._getQtBinDirs()), [], ) count = 0 for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in ("libeay32.dll", "ssleay32.dll"): yield self.makeDllEntryPoint( source_path=filename, dest_path=basename, package_name=full_name, ) count += 1 self.reportFileCount(full_name, count, section="OpenSSL") if ("qml" in self.getQtPluginsSelected() or "all" in self.getQtPluginsSelected()): qml_plugin_dir = self._getQmlDirectory() qml_target_dir = self._getQmlTargetDir() for filename in self._getQmlFileList(dlls=True): filename_relative = os.path.relpath( filename, qml_plugin_dir) yield self.makeDllEntryPoint(source_path=filename, dest_path=os.path.join( qml_target_dir, filename_relative, ), package_name=full_name # reason="Qt QML plugin DLL", ) # Also copy required OpenGL DLLs on Windows if isWin32Windows(): opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll") count = 0 for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in opengl_dlls or basename.startswith( "d3dcompiler_"): yield self.makeDllEntryPoint( source_path=filename, dest_path=basename, package_name=full_name, ) self.reportFileCount(full_name, count, section="OpenGL") elif full_name == self.binding_name + ".QtNetwork": if not isWin32Windows(): dll_path = self.locateDLL("crypto") if dll_path is not None: yield self.makeDllEntryPoint( source_path=dll_path, dest_path=os.path.basename(dll_path), package_name=full_name, ) dll_path = self.locateDLL("ssl") if dll_path is not None: yield self.makeDllEntryPoint( source_path=dll_path, dest_path=os.path.basename(dll_path), package_name=full_name, ) elif self.isQtWebEngineModule( full_name) and not self.webengine_done_binaries: self.webengine_done_binaries = True # prevent multiple copies self.info("Including QtWebEngine executable.") qt_web_engine_dir = self._getLibraryExecutablePath() for filename, filename_relative in listDir(qt_web_engine_dir): if filename_relative.startswith("QtWebEngineProcess"): yield makeExeEntryPoint( source_path=filename, dest_path=os.path.join(self._getWebEngineTargetDir(), filename_relative), package_name=full_name, ) break else: self.sysexit( "Error, cannot locate QtWebEngineProcess executable at '%s'." % qt_web_engine_dir)
def attachOnefilePayload(dist_dir, onefile_output_filename, start_binary, expect_compression): # Somewhat detail rich, pylint: disable=too-many-locals compression_indicator, compressor = getCompressorFunction( expect_compression=expect_compression) with _openBinaryFileForAppending(onefile_output_filename) as output_file: # Seeking to end of file seems necessary on Python2 at least, maybe it's # just that tell reports wrong value initially. output_file.seek(0, 2) start_pos = output_file.tell() output_file.write(b"KA" + compression_indicator) # Move the binary to start immediately to the start position file_list = getFileList(dist_dir, normalize=False) file_list.remove(start_binary) file_list.insert(0, start_binary) if isWin32Windows(): filename_encoding = "utf-16le" else: filename_encoding = "utf8" payload_size = 0 setupProgressBar( stage="Onefile Payload", unit="module", total=len(file_list), ) with compressor(output_file) as compressed_file: for filename_full in file_list: filename_relative = os.path.relpath(filename_full, dist_dir) reportProgressBar( item=filename_relative, update=False, ) filename_encoded = (filename_relative + "\0").encode(filename_encoding) compressed_file.write(filename_encoded) payload_size += len(filename_encoded) with open(filename_full, "rb") as input_file: input_file.seek(0, 2) input_size = input_file.tell() input_file.seek(0, 0) compressed_file.write(struct.pack("Q", input_size)) shutil.copyfileobj(input_file, compressed_file) payload_size += input_size + 8 reportProgressBar( item=filename_relative, update=True, ) # Using empty filename as a terminator. filename_encoded = "\0".encode(filename_encoding) compressed_file.write(filename_encoded) payload_size += len(filename_encoded) compressed_size = compressed_file.tell() if compression_indicator == b"Y": onefile_logger.info( "Onefile payload compression ratio (%.2f%%) size %d to %d." % ( (float(compressed_size) / payload_size) * 100, payload_size, compressed_size, )) if isWin32Windows(): # add padding to have the start position at a double world boundary # this is needed on windows so that a possible certificate immediately # follows the start position pad = output_file.tell() % 8 if pad != 0: output_file.write(bytes(8 - pad)) output_file.seek(0, 2) end_pos = output_file.tell() # Size of the payload data plus the size of that size storage, so C code can # jump directly to it. output_file.write(struct.pack("Q", end_pos - start_pos)) closeProgressBar()
def considerExtraDlls(self, dist_dir, module): # pylint: disable=too-many-branches,too-many-locals,too-many-statements full_name = module.getFullName() if full_name == self.binding_name: if not self.getQtPluginDirs(): self.sysexit( "Error, failed to detect %r plugin directories." % self.binding_name ) target_plugin_dir = os.path.join(dist_dir, full_name.asPath(), "qt-plugins") self.info( "Copying Qt plug-ins '%s' to '%s'." % ( ",".join( sorted(x for x in self.getQtPluginsSelected() if x != "xml") ), target_plugin_dir, ) ) # TODO: Change this to filtering copyTree while it's doing it. for plugin_dir in self.getQtPluginDirs(): copyTree(plugin_dir, target_plugin_dir) if "all" not in self.getQtPluginsSelected(): for plugin_candidate in getSubDirectories(target_plugin_dir): if ( os.path.basename(plugin_candidate) not in self.getQtPluginsSelected() ): removeDirectory(plugin_candidate, ignore_errors=False) for plugin_candidate in self.getQtPluginsSelected(): if plugin_candidate == "qml": continue if not os.path.isdir( os.path.join(target_plugin_dir, plugin_candidate) ): self.sysexit( "Error, no such Qt plugin family: %s" % plugin_candidate ) result = self.findDLLs( full_name=full_name, target_plugin_dir=target_plugin_dir, ) if isWin32Windows(): # Those 2 vars will be used later, just saving some resources # by caching the files list qt_bin_files = sum( (getFileList(qt_bin_dir) for qt_bin_dir in self._getQtBinDirs()), [], ) self.info("Copying OpenSSL DLLs to %r." % dist_dir) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in ("libeay32.dll", "ssleay32.dll"): shutil.copy(filename, os.path.join(dist_dir, basename)) if ( "qml" in self.getQtPluginsSelected() or "all" in self.getQtPluginsSelected() ): result += self.copyQmlFiles( full_name=full_name, target_plugin_dir=target_plugin_dir, ) # Also copy required OpenGL DLLs on Windows if isWin32Windows(): opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll") self.info("Copying OpenGL DLLs to %r." % dist_dir) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in opengl_dlls or basename.startswith( "d3dcompiler_" ): shutil.copy(filename, os.path.join(dist_dir, basename)) return result elif full_name == self.binding_name + ".QtNetwork": if not isWin32Windows(): dll_path = locateDLL("crypto") if dll_path is not None: dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path)) shutil.copy(dll_path, dist_dll_path) dll_path = locateDLL("ssl") if dll_path is not None: dist_dll_path = os.path.join(dist_dir, os.path.basename(dll_path)) shutil.copy(dll_path, dist_dll_path) elif ( full_name in ( self.binding_name + ".QtWebEngine", self.binding_name + ".QtWebEngineCore", self.binding_name + ".QtWebEngineWidgets", ) and not self.webengine_done ): self.webengine_done = True # prevent multiple copies self.info("Copying QtWebEngine components") plugin_parent = os.path.dirname(self.getQtPluginDirs()[0]) if isWin32Windows(): bin_dir = os.path.join(plugin_parent, "bin") else: # TODO verify this for non-Windows! bin_dir = os.path.join(plugin_parent, "libexec") target_bin_dir = os.path.join(dist_dir) for f in os.listdir(bin_dir): if f.startswith("QtWebEngineProcess"): shutil.copy(os.path.join(bin_dir, f), target_bin_dir) resources_dir = os.path.join(plugin_parent, "resources") target_resources_dir = os.path.join(dist_dir) for f in os.listdir(resources_dir): shutil.copy(os.path.join(resources_dir, f), target_resources_dir) translations_dir = os.path.join(plugin_parent, "translations") pos = len(translations_dir) + 1 target_translations_dir = os.path.join( dist_dir, full_name.getTopLevelPackageName().asPath(), "Qt", "translations", ) for f in getFileList(translations_dir): tar_f = os.path.join(target_translations_dir, f[pos:]) makePath(os.path.dirname(tar_f)) shutil.copyfile(f, tar_f) return ()
def considerExtraDlls(self, dist_dir, module): full_name = module.getFullName() if full_name in ("PyQt4", "PyQt5"): qt_version = int(full_name[-1]) plugin_dir, = self.getPyQtPluginDirs(qt_version) target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins") plugin_options = self.getPluginOptions() plugin_options = set(plugin_options) if not plugin_options: plugin_options.add("sensible") if "sensible" in plugin_options: # Most used ones with low dependencies. plugin_options.update( tuple( family for family in ( "imageformats", "iconengines", "mediaservice", "printsupport", ) if self.hasPluginFamily(plugin_dir, family) ) ) plugin_options.remove("sensible") # Make sure the above didn't detect nothing, which would be # indicating the check to be bad. assert plugin_options # Seems platforms is required on Windows. if os.name == "nt": plugin_options.add("platforms") info( "Copying Qt plug-ins '%s' to '%s'." % ( ",".join(sorted(x for x in plugin_options if x != "xml")), target_plugin_dir, ) ) shutil.copytree(plugin_dir, target_plugin_dir) if "all" not in plugin_options: for plugin_candidate in getSubDirectories(target_plugin_dir): if os.path.basename(plugin_candidate) not in plugin_options: removeDirectory(plugin_candidate, ignore_errors=False) for plugin_candidate in plugin_options: if plugin_candidate == "qml": continue if not os.path.isdir( os.path.join(target_plugin_dir, plugin_candidate) ): sys.exit( "Error, no such Qt plugin family: %s" % plugin_candidate ) result = [ ( filename, os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ), full_name, ) for filename in getFileList(plugin_dir) if not filename.endswith(".qml") if os.path.exists( os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ) ) ] if "qml" in plugin_options or "all" in plugin_options: qml_plugin_dir = os.path.normpath(os.path.join(plugin_dir, "..", "qml")) qml_target_dir = os.path.normpath( os.path.join(target_plugin_dir, "..", "Qt", "qml") ) info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir)) shutil.copytree(qml_plugin_dir, qml_target_dir) # We try to filter here, not for DLLs. result += [ ( filename, os.path.join( qml_target_dir, os.path.relpath(filename, qml_plugin_dir) ), full_name, ) for filename in getFileList(qml_plugin_dir) if not filename.endswith( ( ".qml", ".qmlc", ".qmltypes", ".js", ".jsc", ".png", ".ttf", ".metainfo", ) ) if not os.path.isdir(filename) if not os.path.basename(filename) == "qmldir" ] return result return ()
def considerExtraDlls(self, dist_dir, module): # pylint: disable=too-many-branches,too-many-locals full_name = module.getFullName() if full_name in ("PyQt4", "PyQt5"): qt_version = int(full_name[-1]) plugin_dirs = self.getPyQtPluginDirs(qt_version) if not plugin_dirs: sys.exit("Error, failed to detect %s plugin directories." % full_name) target_plugin_dir = os.path.join(dist_dir, full_name, "qt-plugins") plugin_options = self.getPluginOptions() plugin_options = set(plugin_options) # Default to using sensible plugins. if not plugin_options: plugin_options.add("sensible") if "sensible" in plugin_options: # Most used ones with low dependencies. plugin_options.update( tuple( family for family in ( "imageformats", "iconengines", "mediaservice", "printsupport", "platforms", ) if self.hasPluginFamily(plugin_dirs, family) ) ) plugin_options.remove("sensible") # Make sure the above didn't detect nothing, which would be # indicating the check to be bad. assert plugin_options info( "Copying Qt plug-ins '%s' to '%s'." % ( ",".join(sorted(x for x in plugin_options if x != "xml")), target_plugin_dir, ) ) for plugin_dir in plugin_dirs: copyTree(plugin_dir, target_plugin_dir) if "all" not in plugin_options: for plugin_candidate in getSubDirectories(target_plugin_dir): if os.path.basename(plugin_candidate) not in plugin_options: removeDirectory(plugin_candidate, ignore_errors=False) for plugin_candidate in plugin_options: if plugin_candidate == "qml": continue if not os.path.isdir( os.path.join(target_plugin_dir, plugin_candidate) ): sys.exit( "Error, no such Qt plugin family: %s" % plugin_candidate ) result = [ ( filename, os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ), full_name, ) for plugin_dir in plugin_dirs for filename in getFileList(plugin_dir) if not filename.endswith(".qml") if os.path.exists( os.path.join( target_plugin_dir, os.path.relpath(filename, plugin_dir) ) ) ] if isWin32Windows(): # Those 2 vars will be used later, just saving some resources # by caching the files list qt_bin_files = sum( ( getFileList(qt_bin_dir) for qt_bin_dir in self._getQtBinDirs(plugin_dirs) ), [], ) info("Copying OpenSSL DLLs to %r" % (dist_dir,)) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in ("libeay32.dll", "ssleay32.dll"): shutil.copy(filename, os.path.join(dist_dir, basename)) if "qml" in plugin_options or "all" in plugin_options: for plugin_dir in plugin_dirs: qml_plugin_dir = os.path.normpath( os.path.join(plugin_dir, "..", "qml") ) if os.path.exists(qml_plugin_dir): break else: sys.exit("Error, no such Qt plugin family: qml") qml_target_dir = os.path.normpath( os.path.join(target_plugin_dir, "..", "Qt", "qml") ) info("Copying Qt plug-ins 'xml' to '%s'." % (qml_target_dir)) copyTree(qml_plugin_dir, qml_target_dir) # We try to filter here, not for DLLs. result += [ ( filename, os.path.join( qml_target_dir, os.path.relpath(filename, qml_plugin_dir) ), full_name, ) for filename in getFileList(qml_plugin_dir) if not filename.endswith( ( ".qml", ".qmlc", ".qmltypes", ".js", ".jsc", ".png", ".ttf", ".metainfo", ) ) if not os.path.isdir(filename) if not os.path.basename(filename) == "qmldir" ] # Also copy required OpenGL DLLs on Windows if isWin32Windows(): opengl_dlls = ("libegl.dll", "libglesv2.dll", "opengl32sw.dll") info("Copying OpenGL DLLs to %r" % (dist_dir,)) for filename in qt_bin_files: basename = os.path.basename(filename).lower() if basename in opengl_dlls or basename.startswith( "d3dcompiler_" ): shutil.copy(filename, os.path.join(dist_dir, basename)) return result return ()
def copyDataFiles(dist_dir): """Copy the data files needed for standalone distribution. Args: dist_dir: The distribution folder under creation Notes: This is for data files only, not DLLs or even extension modules, those must be registered as entry points, and would not go through necessary handling if provided like this. """ # Many details to deal with, pylint: disable=too-many-branches,too-many-locals for pattern, dest, arg in Options.getShallIncludeDataFiles(): filenames = resolveShellPatternToFilenames(pattern) if not filenames: inclusion_logger.warning("No match data file to be included: %r" % pattern) for filename in filenames: file_reason = "specified data file %r on command line" % arg rel_path = dest if rel_path.endswith(("/", os.path.sep)): rel_path = os.path.join(rel_path, os.path.basename(filename)) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(filename, rel_path, file_reason), ) for src, dest in Options.getShallIncludeDataDirs(): filenames = getFileList(src) if not filenames: inclusion_logger.warning("No files in directory" % src) for filename in filenames: relative_filename = relpath(filename, src) file_reason = "specified data dir %r on command line" % src rel_path = os.path.join(dest, relative_filename) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(filename, rel_path, file_reason), ) # Cyclic dependency from nuitka import ModuleRegistry for module in ModuleRegistry.getDoneModules(): for plugin, included_datafile in Plugins.considerDataFiles(module): _handleDataFile(dist_dir=dist_dir, tracer=plugin, included_datafile=included_datafile) for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonPackage( ) or module.isUncompiledPythonPackage(): package_name = module.getFullName() match, reason = package_name.matchesToShellPatterns( patterns=Options.getShallIncludePackageData()) if match: package_directory = module.getCompileTimeDirectory() pkg_filenames = getFileList( package_directory, ignore_dirs=("__pycache__", ), ignore_suffixes=(".py", ".pyw", ".pyc", ".pyo", ".dll") + getSharedLibrarySuffixes(), ) if pkg_filenames: file_reason = "package '%s' %s" % (package_name, reason) for pkg_filename in pkg_filenames: rel_path = os.path.join( package_name.asPath(), os.path.relpath(pkg_filename, package_directory), ) _handleDataFile( dist_dir, inclusion_logger, makeIncludedDataFile(pkg_filename, rel_path, file_reason), )
def _getSubDirectoryFiles2(module, subdirs, folders_only): """Get filenames or dirnames in given subdirs of the module. Notes: All filenames in folders below one of the subdirs are recursively retrieved and returned shortened to begin with the string of subdir. Args: module: module object subdirs: sub folder name(s) - str or None or tuple folders_only: (bool) indicate, whether just the folder structure should be generated. In that case, an empty file named DUMMY will be placed in each of these folders. Yields: Tuples of paths (source, dest) are yielded if folders_only is False, else IncludedDataFile instances for empty dirs. """ module_dir = module.getCompileTimeDirectory() file_list = [] if subdirs is None: data_dirs = [module_dir] elif isinstance(subdirs, basestring): data_dirs = [os.path.join(module_dir, subdirs)] else: data_dirs = [os.path.join(module_dir, subdir) for subdir in subdirs] # Gather the full file list, probably makes no sense to include bytecode files file_list = sum( (getFileList(data_dir, ignore_dirs=("__pycache__", ), ignore_suffixes=(".pyc", )) for data_dir in data_dirs), [], ) if not file_list: msg = "No files or folders found for '%s' in subfolder(s) %r (%r)." % ( module.getFullName(), subdirs, data_dirs, ) NuitkaPluginDataFileCollector.warning(msg) is_package = module.isCompiledPythonPackage( ) or module.isUncompiledPythonPackage() # We need to preserve the package target path in the dist folder. if is_package: package_part = module.getFullName().asPath() else: package = module.getFullName().getPackageName() if package is None: package_part = "" else: package_part = package.asPath() item_set = OrderedSet() for f in file_list: target = os.path.join(package_part, os.path.relpath(f, module_dir)) if folders_only: dir_name = os.path.dirname(target) item_set.add(dir_name) else: item_set.add((f, target)) if folders_only: return makeIncludedEmptyDirectories( source_path=module_dir, dest_paths=item_set, reason="Subdirectories of module %s" % module.getFullName(), ) return item_set