def _detectedPrecompiledFile(filename, module_name, result, user_provided, technical): if filename.endswith(".pyc"): if os.path.isfile(filename[:-1]): return _detectedSourceFile( filename=filename[:-1], module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) if module_name in module_names: return if Options.isShowInclusion(): inclusion_logger.info("Freezing module '%s' (from '%s')." % (module_name, filename)) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=loadCodeObjectData(precompiled_filename=filename), is_package="__init__" in filename, filename=filename, user_provided=user_provided, technical=technical, ) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)
def _copyDllsUsed(dist_dir, used_dlls): setupProgressBar( stage="Copying used DLLs", unit="DLL", total=len(used_dlls), ) dll_map = [] for dll_filename, (package_name, sources) in iterItems(used_dlls): dll_name = os.path.basename(dll_filename) target_path = os.path.join(dist_dir, dll_name) reportProgressBar(target_path) # Sometimes DLL dependencies were copied there already. TODO: That should # actually become disallowed with plugins no longer seeing that folder. if not os.path.exists(target_path): copyDllFile(source_path=dll_filename, dest_path=target_path) dll_map.append((dll_filename, package_name, dll_name)) if Options.isShowInclusion(): inclusion_logger.info( "Included used shared library '%s' (used by %s)." % (dll_filename, ", ".join(sources))) closeProgressBar() return dll_map
def getMetapathLoaderBodyCode(bytecode_accessor): metapath_loader_inittab = [] metapath_module_decls = [] for other_module in getDoneModules(): metapath_loader_inittab.append( getModuleMetapathLoaderEntryCode( module=other_module, bytecode_accessor=bytecode_accessor ) ) if other_module.isCompiledPythonModule(): metapath_module_decls.append( """\ extern PyObject *modulecode_%(module_identifier)s(PyObject *, struct Nuitka_MetaPathBasedLoaderEntry const *);""" % {"module_identifier": other_module.getCodeName()} ) for uncompiled_module in getUncompiledModules(): metapath_loader_inittab.append( getModuleMetapathLoaderEntryCode( module=uncompiled_module, bytecode_accessor=bytecode_accessor ) ) frozen_defs = [] for uncompiled_module in getUncompiledTechnicalModules(): module_name = uncompiled_module.getFullName() code_data = uncompiled_module.getByteCode() is_package = uncompiled_module.isUncompiledPythonPackage() size = len(code_data) # Packages are indicated with negative size. if is_package: size = -size accessor_code = bytecode_accessor.getBlobDataCode(code_data) frozen_defs.append( """\ {{"{module_name}", {start}, {size}}},""".format( module_name=module_name, start=accessor_code[accessor_code.find("[") + 1 : -1], size=size, ) ) if Options.isShowInclusion(): inclusion_logger.info("Embedded as frozen module '%s'." % module_name) return template_metapath_loader_body % { "metapath_module_decls": indented(metapath_module_decls, 0), "metapath_loader_inittab": indented(metapath_loader_inittab), "bytecode_count": bytecode_accessor.getConstantsCount(), "frozen_modules": indented(frozen_defs), }
def setSharedLibraryRPATH(filename, rpath): if Options.isShowInclusion(): inclusion_logger.info("Setting 'RPATH' value '%s' for '%s'." % (rpath, filename)) with withMadeWritableFileMode(filename): if isMacOS(): _setSharedLibraryRPATHDarwin(filename, rpath) else: _setSharedLibraryRPATHElf(filename, rpath)
def removeSharedLibraryRPATH(filename): rpath = getSharedLibraryRPATH(filename) if rpath is not None: if Options.isShowInclusion(): inclusion_logger.info("Removing 'RPATH' setting '%s' from '%s'." % (rpath, filename)) with withMadeWritableFileMode(filename): if getOS() == "Darwin": return _removeSharedLibraryRPATHDarwin(filename, rpath) else: return _removeSharedLibraryRPATHElf(filename)
def demoteCompiledModuleToBytecode(module): """Demote a compiled module to uncompiled (bytecode).""" full_name = module.getFullName() filename = module.getCompileTimeFilename() if Options.isShowProgress(): inclusion_logger.info("Demoting module %r to bytecode from %r." % (full_name, filename)) source_code = module.getSourceCode() # Second chance for plugins to modify source code just before turning it # to bytecode. source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name, is_package=False, source_code=source_code) bytecode = compile(source_code, filename, "exec", dont_inherit=True) bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name, is_package=False, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=full_name, filename=filename, bytecode=marshal.dumps(bytecode), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) uncompiled_module.setUsedModules(module.getUsedModules()) module.finalize() if isImportedModuleByName(full_name): replaceImportedModule(old=module, new=uncompiled_module) replaceRootModule(old=module, new=uncompiled_module) from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule if isTriggerModule(module): replaceTriggerModule(old=module, new=uncompiled_module)
def demoteCompiledModuleToBytecode(module): """Demote a compiled module to uncompiled (bytecode).""" full_name = module.getFullName() filename = module.getCompileTimeFilename() if Options.isShowProgress(): inclusion_logger.info("Demoting module %r to bytecode from %r." % (full_name.asString(), filename)) source_code = module.getSourceCode() bytecode = demoteSourceCodeToBytecode(module_name=full_name, source_code=source_code, filename=filename) uncompiled_module = makeUncompiledPythonModule( module_name=full_name, filename=filename, bytecode=bytecode, is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) used_modules = module.getUsedModules() uncompiled_module.setUsedModules(used_modules) module.finalize() if isImportedModuleByName(full_name): replaceImportedModule(old=module, new=uncompiled_module) replaceRootModule(old=module, new=uncompiled_module) from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule if isTriggerModule(module): replaceTriggerModule(old=module, new=uncompiled_module) writeImportedModulesNamesToCache(module_name=full_name, source_code=source_code, used_modules=used_modules)
def _detectedSourceFile(filename, module_name, result, user_provided, technical): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename=filename, module_name=ModuleName("_collections_abc"), result=result, user_provided=user_provided, technical=technical, ) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = "\n" + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code, ) # Debian stretch site.py source_code = source_code.replace( "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []") # Anaconda3 4.1.2 site.py source_code = source_code.replace( "def main():", "def main():return\n\nif 0:\n def _unused():") if Options.isShowInclusion(): inclusion_logger.info("Freezing module '%s' (from '%s')." % (module_name, filename)) is_package = os.path.basename(filename) == "__init__.py" # Plugins can modify source code: source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name, is_package=is_package, source_code=source_code) bytecode = compileSourceToBytecode( source_code=source_code, filename=module_name.replace(".", os.path.sep) + ".py", ) # Plugins can modify bytecode code: bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name, is_package=is_package, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(bytecode), is_package=is_package, filename=filename, user_provided=user_provided, technical=technical, ) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)
def copyUsedDLLs(source_dir, dist_dir, standalone_entry_points): # This is terribly complex, because we check the list of used DLLs # trying to avoid duplicates, and detecting errors with them not # being binary identical, so we can report them. And then of course # we also need to handle OS specifics. # pylint: disable=too-many-branches,too-many-locals,too-many-statements used_dlls = detectUsedDLLs( source_dir=source_dir, standalone_entry_points=standalone_entry_points, use_cache=not Options.shallNotUseDependsExeCachedResults() and not Options.getWindowsDependencyTool() == "depends.exe", update_cache=not Options.shallNotStoreDependsExeCachedResults() and not Options.getWindowsDependencyTool() == "depends.exe", ) removed_dlls = set() warned_about = set() # Fist make checks and remove some. for dll_filename1, sources1 in tuple(iterItems(used_dlls)): if dll_filename1 in removed_dlls: continue for dll_filename2, sources2 in tuple(iterItems(used_dlls)): if dll_filename1 == dll_filename2: continue if dll_filename2 in removed_dlls: continue # Colliding basenames are an issue to us. if os.path.basename(dll_filename1) != os.path.basename( dll_filename2): continue # May already have been removed earlier if dll_filename1 not in used_dlls: continue if dll_filename2 not in used_dlls: continue dll_name = os.path.basename(dll_filename1) if Options.isShowInclusion(): inclusion_logger.info( """Colliding DLL names for %s, checking identity of \ '%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2)) # Check that if a DLL has the same name, if it's identical, then it's easy. if haveSameFileContents(dll_filename1, dll_filename2): del used_dlls[dll_filename2] removed_dlls.add(dll_filename2) continue # For Win32 we can check out file versions. if Utils.isWin32Windows(): dll_version1 = getWindowsDLLVersion(dll_filename1) dll_version2 = getWindowsDLLVersion(dll_filename2) if dll_version2 < dll_version1: del used_dlls[dll_filename2] removed_dlls.add(dll_filename2) solved = True elif dll_version1 < dll_version2: del used_dlls[dll_filename1] removed_dlls.add(dll_filename1) solved = True else: solved = False if solved: if dll_name not in warned_about and dll_name not in ms_runtime_dlls: warned_about.add(dll_name) inclusion_logger.warning( "Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best." % dll_name) continue # So we have conflicting DLLs, in which case we do report the fact. inclusion_logger.warning("""\ Ignoring non-identical DLLs for '%s'. %s used by: %s different from %s used by %s""" % ( dll_name, dll_filename1, "\n ".join(sources1), dll_filename2, "\n ".join(sources2), )) del used_dlls[dll_filename2] removed_dlls.add(dll_filename2) dll_map = [] for dll_filename, sources in iterItems(used_dlls): dll_name = os.path.basename(dll_filename) target_path = os.path.join(dist_dir, dll_name) shutil.copyfile(dll_filename, target_path) dll_map.append((dll_filename, dll_name)) if Options.isShowInclusion(): inclusion_logger.info( "Included used shared library '%s' (used by %s)." % (dll_filename, ", ".join(sources))) if Utils.getOS() == "Darwin": # For macOS, the binary and the DLLs needs to be changed to reflect # the relative DLL location in the ".dist" folder. for standalone_entry_point in standalone_entry_points: fixupBinaryDLLPathsMacOS( binary_filename=standalone_entry_point.dest_path, dll_map=dll_map, original_location=standalone_entry_point.source_path, ) for original_path, dll_filename in dll_map: fixupBinaryDLLPathsMacOS( binary_filename=os.path.join(dist_dir, dll_filename), dll_map=dll_map, original_location=original_path, ) # Remove code signature from CPython installed library candidate = os.path.join( dist_dir, "Python", ) if os.path.exists(candidate): removeMacOSCodeSignature(candidate) # Remove rpath settings. if Utils.getOS() in ("Linux", "Darwin"): # For Linux, the "rpath" of libraries may be an issue and must be # removed. if Utils.getOS() == "Darwin": start = 0 else: start = 1 for standalone_entry_point in standalone_entry_points[start:]: removeSharedLibraryRPATH(standalone_entry_point.dest_path) for _original_path, dll_filename in dll_map: removeSharedLibraryRPATH(os.path.join(dist_dir, dll_filename)) if Utils.isWin32Windows(): if python_version < 0x300: # For Win32, we might have to remove SXS paths for standalone_entry_point in standalone_entry_points[1:]: removeSxsFromDLL(standalone_entry_point.dest_path) for _original_path, dll_filename in dll_map: removeSxsFromDLL(os.path.join(dist_dir, dll_filename))
def makeSourceDirectory(): """Get the full list of modules imported, create code for all of them.""" # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=too-many-branches # assert main_module in ModuleRegistry.getDoneModules() # We might have chosen to include it as bytecode, and only compiled it for # fun, and to find its imports. In this case, now we just can drop it. Or # a module may shadow a frozen module, but be a different one, then we can # drop the frozen one. # TODO: This really should be done when the compiled module comes into # existence. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): uncompiled_module = ModuleRegistry.getUncompiledModule( module_name=module.getFullName(), module_filename=module.getCompileTimeFilename(), ) if uncompiled_module is not None: # We now need to decide which one to keep, compiled or uncompiled # module. Some uncompiled modules may have been asked by the user # or technically required. By default, frozen code if it exists # is preferred, as it will be from standalone mode adding it. if ( uncompiled_module.isUserProvided() or uncompiled_module.isTechnical() ): ModuleRegistry.removeDoneModule(module) else: ModuleRegistry.removeUncompiledModule(uncompiled_module) # Lets check if the asked modules are actually present, and warn the # user if one of those was not found. for any_case_module in Options.getShallFollowModules(): if "*" in any_case_module or "{" in any_case_module: continue for module in ModuleRegistry.getDoneModules(): if module.getFullName() == any_case_module: break else: general.warning( "Did not follow import to unused '%s', consider include options." % any_case_module ) # Prepare code generation, i.e. execute finalization for it. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): Finalization.prepareCodeGeneration(module) # Do some reporting and determine compiled module to work on compiled_modules = [] for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): compiled_modules.append(module) if Options.isShowInclusion(): inclusion_logger.info( "Included compiled module '%s'." % module.getFullName() ) elif module.isPythonExtensionModule(): addExtensionModuleEntryPoint(module) if Options.isShowInclusion(): inclusion_logger.info( "Included extension module '%s'." % module.getFullName() ) elif module.isUncompiledPythonModule(): if Options.isShowInclusion(): inclusion_logger.info( "Included uncompiled module '%s'." % module.getFullName() ) else: assert False, module # Pick filenames. source_dir = OutputDirectories.getSourceDirectoryPath() module_filenames = pickSourceFilenames( source_dir=source_dir, modules=compiled_modules ) setupProgressBar( stage="C Source Generation", unit="module", total=len(compiled_modules), ) # Generate code for compiled modules, this can be slow, so do it separately # with a progress bar. for module in compiled_modules: c_filename = module_filenames[module] reportProgressBar( item=module.getFullName(), ) source_code = CodeGeneration.generateModuleCode( module=module, data_filename=os.path.basename(c_filename + "onst"), # Really .const ) writeSourceCode(filename=c_filename, source_code=source_code) closeProgressBar() ( helper_decl_code, helper_impl_code, constants_header_code, constants_body_code, ) = CodeGeneration.generateHelpersCode() writeSourceCode( filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code ) writeSourceCode( filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code ) writeSourceCode( filename=os.path.join(source_dir, "__constants.h"), source_code=constants_header_code, ) writeSourceCode( filename=os.path.join(source_dir, "__constants.c"), source_code=constants_body_code, )
def makeSourceDirectory(main_module): """Get the full list of modules imported, create code for all of them.""" # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=too-many-branches assert main_module.isCompiledPythonModule() # assert main_module in ModuleRegistry.getDoneModules() # We might have chosen to include it as bytecode, and only compiled it for # fun, and to find its imports. In this case, now we just can drop it. Or # a module may shadow a frozen module, but be a different one, then we can # drop the frozen one. # TODO: This really should be done when the compiled module comes into # existence. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): uncompiled_module = ModuleRegistry.getUncompiledModule( module_name=module.getFullName(), module_filename=module.getCompileTimeFilename(), ) if uncompiled_module is not None: # We now need to decide which one to keep, compiled or uncompiled # module. Some uncompiled modules may have been asked by the user # or technically required. By default, frozen code if it exists # is preferred, as it will be from standalone mode adding it. if ( uncompiled_module.isUserProvided() or uncompiled_module.isTechnical() ): ModuleRegistry.removeDoneModule(module) else: ModuleRegistry.removeUncompiledModule(uncompiled_module) # Lets check if the recurse-to modules are actually present, and warn the # user if one of those was not found. for any_case_module in Options.getShallFollowModules(): if "*" in any_case_module or "{" in any_case_module: continue for module in ModuleRegistry.getDoneModules(): if module.getFullName() == any_case_module: break else: general.warning("Not recursing to unused '%s'." % any_case_module) # Prepare code generation, i.e. execute finalization for it. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): Finalization.prepareCodeGeneration(module) # Pick filenames. source_dir = OutputDirectories.getSourceDirectoryPath() module_filenames = pickSourceFilenames( source_dir=source_dir, modules=ModuleRegistry.getDoneModules() ) # Generate code for modules. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): c_filename = module_filenames[module] source_code = CodeGeneration.generateModuleCode( module=module, data_filename=os.path.basename(c_filename + "onst"), # Really .const ) writeSourceCode(filename=c_filename, source_code=source_code) if Options.isShowInclusion(): inclusion_logger.info( "Included compiled module '%s'." % module.getFullName() ) elif module.isPythonShlibModule(): addShlibEntryPoint(module) if Options.isShowInclusion(): inclusion_logger.info( "Included extension module '%s'." % module.getFullName() ) elif module.isUncompiledPythonModule(): if Options.isShowInclusion(): inclusion_logger.info( "Included uncompiled module '%s'." % module.getFullName() ) else: assert False, module ( helper_decl_code, helper_impl_code, constants_header_code, constants_body_code, ) = CodeGeneration.generateHelpersCode() writeSourceCode( filename=os.path.join(source_dir, "__helpers.h"), source_code=helper_decl_code ) writeSourceCode( filename=os.path.join(source_dir, "__helpers.c"), source_code=helper_impl_code ) writeSourceCode( filename=os.path.join(source_dir, "__constants.h"), source_code=constants_header_code, ) writeSourceCode( filename=os.path.join(source_dir, "__constants.c"), source_code=constants_body_code, ) for filename, source_code in Plugins.getExtraCodeFiles().items(): target_dir = os.path.join(source_dir, "plugins") if not os.path.isdir(target_dir): makePath(target_dir) writeSourceCode( filename=os.path.join(target_dir, filename), source_code=source_code )
def _removeDuplicateDlls(used_dlls): # Many things to consider, pylint: disable=too-many-branches removed_dlls = set() warned_about = set() # Identical DLLs are interesting for DLL resolution on macOS at least. duplicate_dlls = {} # Fist make checks and remove some, in loops we copy the items so we can remove # the used_dll list freely. for dll_filename1, (_package_name1, sources1) in tuple(iterItems(used_dlls)): if dll_filename1 in removed_dlls: continue for dll_filename2, (_package_name1, sources2) in tuple(iterItems(used_dlls)): if dll_filename1 == dll_filename2: continue if dll_filename2 in removed_dlls: continue # Colliding basenames are an issue to us. if os.path.basename(dll_filename1) != os.path.basename( dll_filename2): continue # May already have been removed earlier if dll_filename1 not in used_dlls: continue if dll_filename2 not in used_dlls: continue dll_name = os.path.basename(dll_filename1) if Options.isShowInclusion(): inclusion_logger.info( """Colliding DLL names for %s, checking identity of \ '%s' <-> '%s'.""" % (dll_name, dll_filename1, dll_filename2)) # Check that if a DLL has the same name, if it's identical, then it's easy. if haveSameFileContents(dll_filename1, dll_filename2): del used_dlls[dll_filename2] removed_dlls.add(dll_filename2) duplicate_dlls.setdefault(dll_filename1, []).append(dll_filename2) duplicate_dlls.setdefault(dll_filename2, []).append(dll_filename1) continue # For Win32 we can check out file versions. if Utils.isWin32Windows(): dll_version1 = getWindowsDLLVersion(dll_filename1) dll_version2 = getWindowsDLLVersion(dll_filename2) if dll_version2 < dll_version1: del used_dlls[dll_filename2] removed_dlls.add(dll_filename2) solved = True elif dll_version1 < dll_version2: del used_dlls[dll_filename1] removed_dlls.add(dll_filename1) solved = True else: solved = False if solved: if dll_name not in warned_about and dll_name not in ms_runtime_dlls: warned_about.add(dll_name) inclusion_logger.warning( "Conflicting DLLs for '%s' in your installation, newest file version used, hoping for the best." % dll_name) continue # So we have conflicting DLLs, in which case we do report the fact. inclusion_logger.warning("""\ Ignoring non-identical DLLs for '%s'. %s used by: %s different from %s used by %s""" % ( dll_name, dll_filename1, "\n ".join(sources1), dll_filename2, "\n ".join(sources2), )) del used_dlls[dll_filename2] removed_dlls.add(dll_filename2) return duplicate_dlls