def _detectedSourceFile(filename, module_name, result, is_late): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile(filename=filename, module_name="_collections_abc", result=result, is_late=is_late) source_code = readSourceCodeFromFilename(filename) if Utils.python_version >= 300: filename = filename.decode("utf-8") if module_name == "site": source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code) debug("Freezing module '%s' (from '%s').", module_name, filename) result.append( (module_name, marshal.dumps(compile(source_code, filename, "exec")), Utils.basename(filename) == "__init__.py", filename, is_late)) module_names.add(module_name)
def checkRequirements(filename): for line in readSourceCodeFromFilename(None, filename).splitlines(): if line.startswith("# nuitka-skip-unless-"): if line[21:33] == "expression: ": expression = line[33:] with open(os.devnull, "w") as devnull: result = subprocess.call( ( os.environ["PYTHON"], "-c", "import sys, os; sys.exit(not bool(%s))" % expression, ), stdout=devnull, stderr=subprocess.STDOUT, ) if result != 0: return (False, "Expression '%s' evaluated to false" % expression) elif line[21:30] == "imports: ": imports_needed = line[30:].rstrip().split(",") for i in imports_needed: if not hasModule(i): return ( False, i + " not installed for this Python version, but test needs it", ) # default return value return (True, "")
def _detectedSourceFile(filename, module_name, result, user_provided): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile(filename=filename, module_name="_collections_abc", result=result, user_provided=user_provided) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, Utils.basename(filename), source_code) debug("Freezing module '%s' (from '%s').", module_name, filename) result.append( makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(compile(source_code, filename, "exec")), is_package=Utils.basename(filename) == "__init__.py", filename=filename, user_provided=user_provided)) module_names.add(module_name)
def demoteCompiledModuleToBytecode(module): """ Demote a compiled module to uncompiled (bytecode). """ full_name = module.getFullName() filename = module.getCompileTimeFilename() source_code = readSourceCodeFromFilename(full_name, filename) source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name, is_package=False, source_code=source_code) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name, is_package=False, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=filename, bytecode=marshal.dumps(bytecode), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False) replaceImportedModule(old=module, new=uncompiled_module) replaceRootModule(old=module, new=uncompiled_module) assert module.constraint_collection is not None uncompiled_module.setUsedModules( module.constraint_collection.getUsedModules())
def getSourceCode(self): if self.source_code is not None: return self.source_code else: return readSourceCodeFromFilename( module_name=self.getFullName(), source_filename=self.getCompileTimeFilename(), )
def _detectedSourceFile(filename, module_name, result, user_provided, technical): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile(filename=filename, module_name="_collections_abc", result=result, user_provided=user_provided, technical=technical) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code) # Debian stretch site.py source_code = source_code.replace( "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []") # Anaconda3 4.1.2 site.py source_code = source_code.replace( "def main():", "def main():return\n\nif 0:\n def _unused():", ) debug("Freezing module '%s' (from '%s').", module_name, filename) is_package = os.path.basename(filename) == "__init__.py" source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name, is_package=is_package, source_code=source_code) bytecode = compile(source_code, filename, "exec", dont_inherit=True) bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name, is_package=is_package, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(bytecode), is_package=is_package, filename=filename, user_provided=user_provided, technical=technical) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)
def recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building if not ImportCache.isImportedModuleByPath(module_relpath): module, source_ref, source_filename = Building.decideModuleTree( filename=module_filename, package=module_package, is_top=False, is_main=False, is_shlib=module_kind == "shlib") # Check if the module name is known. In order to avoid duplicates, # learn the new filename, and continue build if its not. if not ImportCache.isImportedModuleByName(module.getFullName()): debug("Recurse to import '%s' from %s. (%s)", module.getFullName(), module_relpath, reason) if module_kind == "py" and source_filename is not None: try: Building.createModuleTree( module=module, source_ref=source_ref, source_code=readSourceCodeFromFilename( source_filename), is_main=False) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__) return None, False ImportCache.addImportedModule(module_relpath, module) is_added = True else: ImportCache.addImportedModule( module_relpath, ImportCache.getImportedModuleByName(module.getFullName())) module = ImportCache.getImportedModuleByName(module.getFullName()) is_added = False assert not module_relpath.endswith("/__init__.py"), module return module, is_added else: return ImportCache.getImportedModuleByPath(module_relpath), False
def _detectedSourceFile(filename, module_name, result, is_late): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename = filename, module_name = "_collections_abc", result = result, is_late = is_late ) source_code = readSourceCodeFromFilename(module_name, filename) if Utils.python_version >= 300: filename = filename.decode("utf-8") if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, Utils.basename(filename), source_code ) debug( "Freezing module '%s' (from '%s').", module_name, filename ) result.append( ( module_name, marshal.dumps( compile(source_code, filename, "exec") ), Utils.basename(filename) == "__init__.py", filename, is_late ) ) module_names.add(module_name)
def demoteCompiledModuleToBytecode(module): """ Demote a compiled module to uncompiled (bytecode). """ full_name = module.getFullName() filename = module.getCompileTimeFilename() source_code = readSourceCodeFromFilename(full_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name = full_name, is_package = False, source_code = source_code ) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = full_name, is_package = False, bytecode = bytecode ) uncompiled_module = makeUncompiledPythonModule( module_name = module.getFullName(), filename = filename, bytecode = marshal.dumps(bytecode), is_package = module.isCompiledPythonPackage(), user_provided = True, technical = False ) replaceImportedModule( old = module, new = uncompiled_module ) replaceRootModule( old = module, new = uncompiled_module ) assert module.constraint_collection is not None uncompiled_module.setUsedModules(module.constraint_collection.getUsedModules())
def _detectedSourceFile(filename, module_name, result, user_provided): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename = filename, module_name = "_collections_abc", result = result, user_provided = user_provided ) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = '\n' + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, Utils.basename(filename), source_code ) debug( "Freezing module '%s' (from '%s').", module_name, filename ) result.append( makeUncompiledPythonModule( module_name = module_name, bytecode = marshal.dumps( compile(source_code, filename, "exec") ), is_package = Utils.basename(filename) == "__init__.py", filename = filename, user_provided = user_provided ) ) module_names.add(module_name)
def createFakeModuleDependency(module): full_name = module.getFullName() if full_name != "multiprocessing": return # First, build the module node and then read again from the # source code. root_module = getRootTopModule() module_name = ModuleName("__parents_main__") source_code = readSourceCodeFromFilename(module_name, root_module.getFilename()) # For the call stack, this may look bad or different to what # CPython does. Using the "__import__" built-in to not spoil # or use the module namespace. The forking module was split up # into multiple modules in Python 3.4. if python_version >= 0x340: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] # Not needed, and can crash from minor __file__ differences, depending on invocation __import__("multiprocessing.spawn").spawn._fixup_main_from_path = lambda mod_name : None __import__("multiprocessing.spawn").spawn.freeze_support()""" else: source_code += """ __import__("sys").modules["__main__"] = __import__("sys").modules[__name__] __import__("multiprocessing.forking").forking.freeze_support()""" yield ( module_name, source_code, root_module.getCompileTimeFilename(), "Autoenable multiprocessing freeze support", )
def recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule if not ImportCache.isImportedModuleByPath(module_relpath): module, source_ref, source_filename = Building.decideModuleTree( filename = module_filename, package = module_package, is_top = False, is_main = False, is_shlib = module_kind == "shlib" ) # Check if the module name is known. In order to avoid duplicates, # learn the new filename, and continue build if its not. if not ImportCache.isImportedModuleByName(module.getFullName()): debug( "Recurse to import '%s' from %s. (%s)", module.getFullName(), module_relpath, reason ) if module_kind == "py" and source_filename is not None: try: Building.createModuleTree( module = module, source_ref = source_ref, source_code = readSourceCodeFromFilename( module_name = module.getFullName(), source_filename = source_filename ), is_main = False ) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__ ) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because code is too complex.""", module_relpath, module_filename, ) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name = module.getFullName(), filename = module_filename, bytecode = marshal.dumps( compile( readSourceCodeFromFilename(module.getFullName(), module_filename), module_filename, "exec" ) ), is_package = module.isCompiledPythonPackage(), user_provided = True, technical = False ) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) is_added = True else: module = ImportCache.getImportedModuleByName( module.getFullName() ) is_added = False assert not module_relpath.endswith("/__init__.py"), module return module, is_added else: return ImportCache.getImportedModuleByPath(module_relpath), False
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule module, source_ref, source_filename = Building.decideModuleTree( filename=module_filename, package=module_package, is_top=False, is_main=False, is_shlib=module_kind == "shlib", ) logRecursion( "Recurse to import '%s' from '%s'. (%s)", module.getFullName(), module_relpath, reason, ) if module_kind == "py" and source_filename is not None: try: source_code = readSourceCodeFromFilename( module_name=module.getFullName(), source_filename=source_filename ) Building.createModuleTree( module=module, source_ref=source_ref, source_code=source_code, is_main=False, ) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__, ) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because code is too complex.""", module_relpath, module_filename, ) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=module_filename, bytecode=marshal.dumps( compile( source_code, module_filename, "exec", dont_inherit=True ) ), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) return module, True
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule from nuitka.tree import Building module, source_ref, source_filename = Building.decideModuleTree( filename=module_filename, package=module_package, is_top=False, is_main=False, is_shlib=module_kind == "shlib", ) if Options.isShowInclusion(): recursion_logger.info("Recurse to import '%s' from '%s'. (%s)" % (module.getFullName(), module_relpath, reason)) if module_kind == "py" and source_filename is not None: try: source_code = readSourceCodeFromFilename( module_name=module.getFullName(), source_filename=source_filename) Building.createModuleTree( module=module, source_ref=source_ref, source_code=source_code, is_main=False, ) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__, ) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because code is too complex.""", module_relpath, module_filename, ) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=module_filename, bytecode=marshal.dumps( compile(source_code, module_filename, "exec", dont_inherit=True)), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False, ) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) return module, True
def recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building if not ImportCache.isImportedModuleByPath(module_relpath): module, source_ref, source_filename = Building.decideModuleTree( filename = module_filename, package = module_package, is_top = False, is_main = False, is_shlib = module_kind == "shlib" ) # Check if the module name is known. In order to avoid duplicates, # learn the new filename, and continue build if its not. if not ImportCache.isImportedModuleByName(module.getFullName()): debug( "Recurse to import '%s' from %s. (%s)", module.getFullName(), module_relpath, reason ) if module_kind == "py" and source_filename is not None: try: Building.createModuleTree( module = module, source_ref = source_ref, source_code = readSourceCodeFromFilename( module_name = module.getFullName(), source_filename = source_filename ), is_main = False ) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__ ) return None, False ImportCache.addImportedModule( module_relpath, module ) is_added = True else: ImportCache.addImportedModule( module_relpath, ImportCache.getImportedModuleByName(module.getFullName()) ) module = ImportCache.getImportedModuleByName( module.getFullName() ) is_added = False assert not module_relpath.endswith("/__init__.py"), module return module, is_added else: return ImportCache.getImportedModuleByPath(module_relpath), False
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason): from nuitka.tree import Building from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule module, source_ref, source_filename = Building.decideModuleTree( filename=module_filename, package=module_package, is_top=False, is_main=False, is_shlib=module_kind == "shlib") # Check if the module name is known. In order to avoid duplicates, # learn the new filename, and continue build if its not. if not ImportCache.isImportedModuleByName(module.getFullName()): logRecursion("Recurse to import '%s' from '%s'. (%s)", module.getFullName(), module_relpath, reason) if module_kind == "py" and source_filename is not None: try: source_code = readSourceCodeFromFilename( module_name=module.getFullName(), source_filename=source_filename) Building.createModuleTree(module=module, source_ref=source_ref, source_code=source_code, is_main=False) except (SyntaxError, IndentationError) as e: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath, module_filename, e.__class__.__name__) return None, False except Building.CodeTooComplexCode: if module_filename not in Importing.warned_about: Importing.warned_about.add(module_filename) warning( """\ Cannot recurse to import module '%s' (%s) because code is too complex.""", module_relpath, module_filename, ) if Options.isStandaloneMode(): module = makeUncompiledPythonModule( module_name=module.getFullName(), filename=module_filename, bytecode=marshal.dumps( compile(source_code, module_filename, "exec", dont_inherit=True)), is_package=module.isCompiledPythonPackage(), user_provided=True, technical=False) ModuleRegistry.addUncompiledModule(module) return None, False ImportCache.addImportedModule(module) is_added = True else: module = ImportCache.getImportedModuleByName(module.getFullName()) is_added = False return module, is_added
def _detectedSourceFile(filename, module_name, result, user_provided, technical): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename=filename, module_name=ModuleName("_collections_abc"), result=result, user_provided=user_provided, technical=technical, ) source_code = readSourceCodeFromFilename(module_name, filename) if module_name == "site": if source_code.startswith("def ") or source_code.startswith("class "): source_code = "\n" + source_code source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code, ) # Debian stretch site.py source_code = source_code.replace( "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []") if Options.isShowInclusion(): inclusion_logger.info("Freezing module '%s' (from '%s')." % (module_name, filename)) is_package = os.path.basename(filename) == "__init__.py" # Plugins can modify source code: source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name, is_package=is_package, source_code=source_code) bytecode = compileSourceToBytecode( source_code=source_code, filename=module_name.replace(".", os.path.sep) + ".py", ) # Plugins can modify bytecode code: bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name, is_package=is_package, bytecode=bytecode) uncompiled_module = makeUncompiledPythonModule( module_name=module_name, bytecode=marshal.dumps(bytecode), is_package=is_package, filename=filename, user_provided=user_provided, technical=technical, ) ImportCache.addImportedModule(uncompiled_module) result.append(uncompiled_module) module_names.add(module_name)