def _detectedSourceFile(filename, module_name, result, user_provided): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile(filename=filename, module_name="_collections_abc", result=result, user_provided=user_provided) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, Utils.basename(filename), source_code) debug("Freezing module '%s' (from '%s').", module_name, filename) result.append( makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(compile(source_code, filename, "exec")), is_package=Utils.basename(filename) == "__init__.py", filename=filename, user_provided=user_provided)) module_names.add(module_name)
def _detectedPrecompiledFile(filename, module_name, result, user_provided, technical): if filename.endswith(".pyc"): if os.path.isfile(filename[:-1]): return _detectedSourceFile( filename=filename[:-1], module_name=module_name, result=result, user_provided=user_provided, technical=technical, ) if module_name in module_names: return if Options.isShowInclusion(): inclusion_logger.info("Freezing module '%s' (from '%s')." % (module_name, filename)) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=loadCodeObjectData(precompiled_filename=filename), is_package="__init__" in filename, filename=filename, user_provided=user_provided, technical=technical, ) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)
def _detectedPrecompiledFile(filename, module_name, result, user_provided, technical): if filename.endswith(".pyc"): if Utils.isFile(filename[:-1]): return _detectedSourceFile( filename = filename[:-1], module_name = module_name, result = result, user_provided = user_provided, technical = technical ) if module_name in module_names: return debug( "Freezing module '%s' (from '%s').", module_name, filename ) result.append( makeUncompiledPythonModule( module_name = module_name, bytecode = loadCodeObjectData( precompiled_filename = filename ), is_package = "__init__" in filename, filename = filename, user_provided = user_provided, technical = technical ) ) module_names.add(module_name)
def demoteCompiledModuleToBytecode(module): """ Demote a compiled module to uncompiled (bytecode). """ full_name = module.getFullName() filename = module.getCompileTimeFilename() source_code = readSourceCodeFromFilename(full_name, filename) source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name, is_package=False, source_code=source_code) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name, is_package=False, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=filename, bytecode=marshal.dumps(bytecode), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False) replaceImportedModule(old=module, new=uncompiled_module) replaceRootModule(old=module, new=uncompiled_module) assert module.constraint_collection is not None uncompiled_module.setUsedModules( module.constraint_collection.getUsedModules())
def _detectedSourceFile(filename, module_name, result, user_provided, technical): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile(filename=filename, module_name="_collections_abc", result=result, user_provided=user_provided, technical=technical) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code) # Debian stretch site.py source_code = source_code.replace( "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []") # Anaconda3 4.1.2 site.py source_code = source_code.replace( "def main():", "def main():return\n\nif 0:\n def _unused():", ) debug("Freezing module '%s' (from '%s').", module_name, filename) is_package = os.path.basename(filename) == "__init__.py" source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name, is_package=is_package, source_code=source_code) bytecode = compile(source_code, filename, "exec", dont_inherit=True) bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name, is_package=is_package, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(bytecode), is_package=is_package, filename=filename, user_provided=user_provided, technical=technical) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)
def demoteCompiledModuleToBytecode(module): """ Demote a compiled module to uncompiled (bytecode). """ full_name = module.getFullName() filename = module.getCompileTimeFilename() if Options.isShowProgress(): info("Demoting module '%s' to bytecode from '%s'." % (full_name, filename)) source_code = module.getSourceCode() # Second chance for plugins to modify source code just before turning it # to bytecode. source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name, is_package=False, source_code=source_code) bytecode = compile(source_code, filename, "exec", dont_inherit=True) bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name, is_package=False, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=full_name, filename=filename, bytecode=marshal.dumps(bytecode), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) uncompiled_module.setUsedModules(module.getUsedModules()) module.finalize() if isImportedModuleByName(full_name): replaceImportedModule(old=module, new=uncompiled_module) replaceRootModule(old=module, new=uncompiled_module) from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule if isTriggerModule(module): replaceTriggerModule(old=module, new=uncompiled_module)
def _detectedSourceFile(filename, module_name, result, user_provided): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename = filename, module_name = "_collections_abc", result = result, user_provided = user_provided ) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, Utils.basename(filename), source_code ) debug( "Freezing module '%s' (from '%s').", module_name, filename ) result.append( makeUncompiledPythonModule( module_name = module_name, bytecode = marshal.dumps( compile(source_code, filename, "exec") ), is_package = Utils.basename(filename) == "__init__.py", filename = filename, user_provided = user_provided ) ) module_names.add(module_name)
def demoteCompiledModuleToBytecode(module): """ Demote a compiled module to uncompiled (bytecode). """ full_name = module.getFullName() filename = module.getCompileTimeFilename() source_code = readSourceCodeFromFilename(full_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name = full_name, is_package = False, source_code = source_code ) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = full_name, is_package = False, bytecode = bytecode ) uncompiled_module = makeUncompiledPythonModule( module_name = module.getFullName(), filename = filename, bytecode = marshal.dumps(bytecode), is_package = module.isCompiledPythonPackage(), user_provided = True, technical = False ) replaceImportedModule( old = module, new = uncompiled_module ) replaceRootModule( old = module, new = uncompiled_module ) assert module.constraint_collection is not None uncompiled_module.setUsedModules(module.constraint_collection.getUsedModules())
def demoteCompiledModuleToBytecode(module): """Demote a compiled module to uncompiled (bytecode).""" full_name = module.getFullName() filename = module.getCompileTimeFilename() if Options.isShowProgress(): inclusion_logger.info("Demoting module %r to bytecode from %r." % (full_name.asString(), filename)) source_code = module.getSourceCode() bytecode = demoteSourceCodeToBytecode(module_name=full_name, source_code=source_code, filename=filename) uncompiled_module = makeUncompiledPythonModule( module_name=full_name, filename=filename, bytecode=bytecode, is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) used_modules = module.getUsedModules() uncompiled_module.setUsedModules(used_modules) module.finalize() if isImportedModuleByName(full_name): replaceImportedModule(old=module, new=uncompiled_module) replaceRootModule(old=module, new=uncompiled_module) from nuitka.plugins.PluginBase import isTriggerModule, replaceTriggerModule if isTriggerModule(module): replaceTriggerModule(old=module, new=uncompiled_module) writeImportedModulesNamesToCache(module_name=full_name, source_code=source_code, used_modules=used_modules)
def _loadUncompiledModuleFromCache(module_name, is_package, source_code, source_ref): result = makeUncompiledPythonModule( module_name=module_name, filename=source_ref.getFilename(), bytecode=demoteSourceCodeToBytecode( module_name=module_name, source_code=source_code, filename=source_ref.getFilename(), ), user_provided=False, technical=False, is_package=is_package, ) used_modules = OrderedSet() for used_module_name, line_number in getCachedImportedModulesNames( module_name=module_name, source_code=source_code): _module_name, module_filename, finding = Importing.locateModule( module_name=used_module_name, parent_package=None, level=0, ) assert _module_name == used_module_name used_modules.add(( used_module_name, module_filename, finding, 0, source_ref.atLineNumber(line_number), )) # assert not is_package, (module_name, used_modules, result, result.getCompileTimeFilename()) result.setUsedModules(used_modules) return result
def _makeModuleBodyTooComplex(module_name, module_filename, source_code, is_package): if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) recursion_logger.warning("""\ Cannot follow import to import module '%r' ('%r') because code is too complex.""" % ( module_name, module_filename, )) module = makeUncompiledPythonModule( module_name=module_name, filename=module_filename, bytecode=marshal.dumps( compile(source_code, module_filename, "exec", dont_inherit=True)), is_package=is_package, user_provided=True, technical=False, ) ModuleRegistry.addUncompiledModule(module)
def recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule if not ImportCache.isImportedModuleByPath(module_relpath): module, source_ref, source_filename = Building.decideModuleTree( filename = module_filename, package = module_package, is_top = False, is_main = False, is_shlib = module_kind == "shlib" ) # Check if the module name is known. In order to avoid duplicates, # learn the new filename, and continue build if its not. if not ImportCache.isImportedModuleByName(module.getFullName()): debug( "Recurse to import '%s' from %s. (%s)", module.getFullName(), module_relpath, reason ) if module_kind == "py" and source_filename is not None: try: Building.createModuleTree( module = module, source_ref = source_ref, source_code = readSourceCodeFromFilename( module_name = module.getFullName(), source_filename = source_filename ), is_main = False ) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__ ) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because code is too complex.""", module_relpath, module_filename, ) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name = module.getFullName(), filename = module_filename, bytecode = marshal.dumps( compile( readSourceCodeFromFilename(module.getFullName(), module_filename), module_filename, "exec" ) ), is_package = module.isCompiledPythonPackage(), user_provided = True, technical = False ) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) is_added = True else: module = ImportCache.getImportedModuleByName( module.getFullName() ) is_added = False assert not module_relpath.endswith("/__init__.py"), module return module, is_added else: return ImportCache.getImportedModuleByPath(module_relpath), False
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule from nuitka.tree import Building module, source_ref, source_code = Building.decideModuleTree( filename=module_filename, package=module_package, is_top=False, is_main=False, is_shlib=module_kind == "shlib", ) if Options.isShowInclusion(): recursion_logger.info("Recurse to import '%s' from '%s'. (%s)" % (module.getFullName(), module_relpath, reason)) if source_code is not None: try: Building.createModuleTree( module=module, source_ref=source_ref, source_code=source_code, is_main=False, ) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) recursion_logger.warning( """\ Cannot follow import to module %r (%r) because of %r""" % (module_relpath, module_filename, e.__class__.__name__)) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) recursion_logger.warning("""\ Cannot recurse to import module %r (%r) because code is too complex.""" % ( module_relpath, module_filename, )) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=module_filename, bytecode=marshal.dumps( compile(source_code, module_filename, "exec", dont_inherit=True)), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) return module, True
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule module, source_ref, source_filename = Building.decideModuleTree( filename=module_filename, package=module_package, is_top=False, is_main=False, is_shlib=module_kind == "shlib") # Check if the module name is known. In order to avoid duplicates, # learn the new filename, and continue build if its not. if not ImportCache.isImportedModuleByName(module.getFullName()): logRecursion("Recurse to import '%s' from '%s'. (%s)", module.getFullName(), module_relpath, reason) if module_kind == "py" and source_filename is not None: try: source_code = readSourceCodeFromFilename( module_name=module.getFullName(), source_filename=source_filename) Building.createModuleTree(module=module, source_ref=source_ref, source_code=source_code, is_main=False) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because code is too complex.""", module_relpath, module_filename, ) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=module_filename, bytecode=marshal.dumps( compile(source_code, module_filename, "exec", dont_inherit=True)), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) is_added = True else: module = ImportCache.getImportedModuleByName(module.getFullName()) is_added = False return module, is_added
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule module, source_ref, source_filename = Building.decideModuleTree( filename=module_filename, package=module_package, is_top=False, is_main=False, is_shlib=module_kind == "shlib", ) logRecursion( "Recurse to import '%s' from '%s'. (%s)", module.getFullName(), module_relpath, reason, ) if module_kind == "py" and source_filename is not None: try: source_code = readSourceCodeFromFilename( module_name=module.getFullName(), source_filename=source_filename ) Building.createModuleTree( module=module, source_ref=source_ref, source_code=source_code, is_main=False, ) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__, ) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because code is too complex.""", module_relpath, module_filename, ) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=module_filename, bytecode=marshal.dumps( compile( source_code, module_filename, "exec", dont_inherit=True ) ), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) return module, True
def _detectedSourceFile(filename, module_name, result, user_provided, technical): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename=filename, module_name=ModuleName("_collections_abc"), result=result, user_provided=user_provided, technical=technical, ) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = "\n" + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code, ) # Debian stretch site.py source_code = source_code.replace( "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []") if Options.isShowInclusion(): inclusion_logger.info("Freezing module '%s' (from '%s')." % (module_name, filename)) is_package = os.path.basename(filename) == "__init__.py" # Plugins can modify source code: source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name, is_package=is_package, source_code=source_code) bytecode = compileSourceToBytecode( source_code=source_code, filename=module_name.replace(".", os.path.sep) + ".py", ) # Plugins can modify bytecode code: bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name, is_package=is_package, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(bytecode), is_package=is_package, filename=filename, user_provided=user_provided, technical=technical, ) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)