def cleanSourceDirectory(source_dir): if Utils.isDir(source_dir): for path, _filename in Utils.listDir(source_dir): if Utils.getExtension(path) in (".cpp", ".hpp", ".c", ".o", ".os", ".obj", ".bin", ".res", ".rc", ".manifest"): Utils.deleteFile(path, True) else: Utils.makePath(source_dir) static_source_dir = Utils.joinpath( source_dir, "static" ) if Utils.isDir(static_source_dir): for path, _filename in Utils.listDir(static_source_dir): if Utils.getExtension(path) in (".o", ".os", ".obj"): Utils.deleteFile(path, True) win32_source_dir = Utils.joinpath( static_source_dir, "win32_ucontext_src" ) if Utils.getOS() == "Windows": Utils.deleteFile( Utils.joinpath(win32_source_dir, "fibers_win32.obj"), False )
def detectPthImportedPackages(): if not hasattr(sys.modules["site"], "getsitepackages"): return () pth_imports = set() for prefix in sys.modules["site"].getsitepackages(): if not Utils.isDir(prefix): continue for path, filename in Utils.listDir(prefix): if filename.endswith(".pth"): try: for line in open(path, "rU"): if line.startswith("import "): if ';' in line: line = line[:line.find(';')] for part in line[7:].split(','): pth_imports.add(part.strip()) except OSError: warning( "Python installation problem, cannot read file '%s'.") return tuple(sorted(pth_imports))
def checkPluginFilenamePattern(pattern): import sys, glob debug( "Checking plug-in pattern '%s':", pattern, ) if Utils.isDir(pattern): sys.exit("Error, pattern cannot be a directory name.") found = False for filename in glob.iglob(pattern): if filename.endswith(".pyc"): continue if not Utils.isFile(filename): continue found = True _checkPluginPath(filename, None) if not found: warning("Didn't match any files against pattern '%s'." % pattern)
def detectPthImportedPackages(): if not hasattr(sys.modules["site"], "getsitepackages"): return () pth_imports = set() for prefix in sys.modules["site"].getsitepackages(): if not Utils.isDir(prefix): continue for path, filename in Utils.listDir(prefix): if filename.endswith(".pth"): try: for line in open(path, "rU"): if line.startswith("import "): if ';' in line: line = line[:line.find(';')] for part in line[7:].split(','): pth_imports.add(part.strip()) except OSError: warning("Python installation problem, cannot read file '%s'.") return tuple(sorted(pth_imports))
def checkPluginPath(plugin_filename, module_package): debug( "Checking top level plug-in path %s %s", plugin_filename, module_package ) plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: # File or package makes a difference, handle that if Utils.isFile(plugin_info[0]) or \ Importing.isPackageDir(plugin_info[0]): _checkPluginPath(plugin_filename, module_package) elif Utils.isDir(plugin_info[0]): for sub_path, sub_filename in Utils.listDir(plugin_info[0]): assert sub_filename != "__init__.py" if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, None) else: warning("Failed to include module from '%s'.", plugin_info[0]) else: warning("Failed to recurse to directory '%s'.", plugin_filename)
def checkPluginPath(plugin_filename, module_package): debug( "Checking top level plug-in path %s %s", plugin_filename, module_package ) plugin_info = considerFilename( module_filename = plugin_filename ) if plugin_info is not None: # File or package makes a difference, handle that if Utils.isFile(plugin_info[0]) or \ Importing.isPackageDir(plugin_info[0]): _checkPluginPath(plugin_filename, module_package) elif Utils.isDir(plugin_info[0]): for sub_path, sub_filename in Utils.listDir(plugin_info[0]): assert sub_filename != "__init__.py" if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, None) else: warning("Failed to include module from '%s'.", plugin_info[0]) else: warning("Failed to recurse to directory '%s'.", plugin_filename)
def considerImplicitImports(self, module, signal_change): """ Consider module imports. You will most likely want to look at "module.getFullName()" to get the fully qualified module or package name. You do not want to overload this method, but rather the things it calls, as the "signal_change" part of this API is not to be cared about. Most prominently "getImplicitImports()". """ for full_name in self.getImplicitImports(module.getFullName()): module_name = full_name.split('.')[-1] module_package = '.'.join(full_name.split('.')[:-1]) or None module_filename = self.locateModule( source_ref = module.getSourceReference(), module_name = module_name, module_package = module_package, ) if module_filename is None: sys.exit( "Error, implicit module '%s' expected by '%s' not found" % ( module_name, module.getFullName() ) ) elif Utils.isDir(module_filename): module_kind = "py" elif module_filename.endswith(".py"): module_kind = "py" elif module_filename.endswith(".so") or \ module_filename.endswith(".pyd"): module_kind = "shlib" else: assert False, module_filename # TODO: This should get back to plug-ins, they should be allowed to # preempt or override the decision. decision, reason = self.decideRecursion( module_filename = module_filename, module_name = module_name, module_package = module_package, module_kind = module_kind ) if decision: self.recurseTo( module_package = module_package, module_filename = module_filename, module_kind = module_kind, reason = reason, signal_change = signal_change ) full_name = module.getFullName() if full_name in post_modules: addUsedModule(post_modules[full_name])
def cleanSourceDirectory(source_dir): if Utils.isDir(source_dir): for path, _filename in Utils.listDir(source_dir): if Utils.getExtension(path) in (".c", ".h", ".o", ".os", ".obj", ".bin", ".res", ".rc", ".S", ".cpp", ".manifest"): Utils.deleteFile(path, True) else: Utils.makePath(source_dir)
def considerImplicitImports(self, module, signal_change): """ Consider module imports. You will most likely want to look at "module.getFullName()" to get the fully qualified module or package name. You do not want to overload this method, but rather the things it calls, as the "signal_change" part of this API is not to be cared about. Most prominently "getImplicitImports()". """ for full_name in self.getImplicitImports(module.getFullName()): module_name = full_name.split('.')[-1] module_package = '.'.join(full_name.split('.')[:-1]) or None module_filename = self.locateModule( source_ref=module.getSourceReference(), module_name=module_name, module_package=module_package, ) if module_filename is None: sys.exit( "Error, implicit module '%s' expected by '%s' not found" % (module_name, module.getFullName())) elif Utils.isDir(module_filename): module_kind = "py" elif module_filename.endswith(".py"): module_kind = "py" elif module_filename.endswith(".so") or \ module_filename.endswith(".pyd"): module_kind = "shlib" else: assert False, module_filename # TODO: This should get back to plug-ins, they should be allowed to # preempt or override the decision. decision, reason = self.decideRecursion( module_filename=module_filename, module_name=module_name, module_package=module_package, module_kind=module_kind) if decision: self.recurseTo(module_package=module_package, module_filename=module_filename, module_kind=module_kind, reason=reason, signal_change=signal_change) full_name = module.getFullName() if full_name in post_modules: addUsedModule(post_modules[full_name])
def isPackageDir(dirname): """ Decide if a directory is a package. Before Python3.3 it's required to have a "__init__.py" file, but then it became impossible to decide, and for extra fun, there is also the extra packages provided via "*.pth" file tricks by "site.py" loading. """ return Utils.isDir(dirname) and ( Utils.python_version >= 330 or Utils.isFile(Utils.joinpath(dirname, "__init__.py")) or isPreloadedPackagePath(dirname) )
def isPackageDir(dirname): """ Decide if a directory is a package. Before Python3.3 it's required to have a "__init__.py" file, but then it became impossible to decide, and for extra fun, there is also the extra packages provided via "*.pth" file tricks by "site.py" loading. """ return Utils.isDir(dirname) and \ ( python_version >= 330 or Utils.isFile(Utils.joinpath(dirname, "__init__.py")) or isPreloadedPackagePath(dirname) )
def considerFilename(module_filename): module_filename = Utils.normpath(module_filename) if Utils.isDir(module_filename): module_filename = Utils.abspath(module_filename) module_name = Utils.basename(module_filename) module_relpath = Utils.relpath(module_filename) return module_filename, module_relpath, module_name elif module_filename.endswith(".py"): module_name = Utils.basename(module_filename)[:-3] module_relpath = Utils.relpath(module_filename) return module_filename, module_relpath, module_name else: return None
def cleanSourceDirectory(source_dir): if Utils.isDir(source_dir): for path, _filename in Utils.listDir(source_dir): if Utils.getExtension(path) in ( ".c", ".h", ".o", ".os", ".obj", ".bin", ".res", ".rc", ".S", ".cpp", ".manifest", ): Utils.deleteFile(path, True) else: Utils.makePath(source_dir)
def considerFilename(module_filename, module_package): assert module_package is None or \ ( type(module_package) is str and module_package != "" ) module_filename = Utils.normpath(module_filename) if Utils.isDir(module_filename): module_filename = Utils.abspath(module_filename) module_name = Utils.basename(module_filename) module_relpath = Utils.relpath(module_filename) return module_filename, module_relpath, module_name elif module_filename.endswith(".py"): module_name = Utils.basename(module_filename)[:-3] module_relpath = Utils.relpath(module_filename) return module_filename, module_relpath, module_name else: return None
def _makeBinaryPathPathDLLSearchEnv(package_name): # Put the PYTHONPATH into the system "PATH", DLLs frequently live in # the package directories. env = os.environ.copy() path = env.get("PATH", "").split(os.pathsep) # Put the "Python.exe" first. At least for WinPython, they put the DLLs # there. path = [sys.prefix] + sys.path + path if package_name is not None: for element in sys.path: candidate = Utils.joinpath(element, package_name) if Utils.isDir(candidate): path.append(candidate) env["PATH"] = os.pathsep.join(path) return env
def _makeBinaryPathPathDLLSearchEnv(package_name): # Put the PYTHONPATH into the system "PATH", DLLs frequently live in # the package directories. env = os.environ.copy() path = env.get("PATH","").split(';') # Put the "Python.exe" first. At least for WinPython, they put the DLLs # there. path = [sys.prefix] + sys.path + path if package_name is not None: for element in sys.path: candidate = Utils.joinpath(element, package_name) if Utils.isDir(candidate): path.append(candidate) env["PATH"] = ';'.join(path) return env
def detectPthImportedPackages(): if not hasattr(sys.modules["site"], "getsitepackages"): return () pth_imports = set() for prefix in sys.modules["site"].getsitepackages(): if not Utils.isDir(prefix): continue for path, filename in Utils.listDir(prefix): if filename.endswith(".pth"): for line in open(path, "rU"): if line.startswith("import "): if ';' in line: line = line[:line.find(';')] for part in line[7:].split(','): pth_imports.add(part.strip()) return tuple(sorted(pth_imports))
def getModuleNameAndKindFromFilename(module_filename): if Utils.isDir(module_filename): module_name = Utils.basename(module_filename) module_kind = "py" elif module_filename.endswith(".py"): module_name = Utils.basename(module_filename)[:-3] module_kind = "py" else: for suffix, _mode, kind in imp.get_suffixes(): if kind != imp.C_EXTENSION: continue if module_filename.endswith(suffix): module_name = Utils.basename(module_filename)[:-len(suffix)] module_kind = "shlib" break else: module_kind = None module_name = None return module_name, module_kind
def _checkPluginPath(plugin_filename, module_package): # Many branches, for the decision is very complex, pylint: disable=R0912 debug( "Checking detail plug-in path '%s' '%s':", plugin_filename, module_package ) plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: module, is_added = recurseTo( module_filename = plugin_info[0], module_relpath = plugin_info[1], module_package = module_package, module_kind = "py", reason = "Lives in plug-in directory." ) if module: if not is_added: warning( "Recursed to %s '%s' at '%s' twice.", "package" if module.isPythonPackage() else "module", module.getName(), plugin_info[0] ) if not isSameModulePath(module.getFilename(), plugin_info[0]): warning( "Duplicate ignored '%s'.", plugin_info[1] ) return debug( "Recursed to %s %s %s", module.getName(), module.getPackage(), module ) if module.isPythonPackage(): package_filename = module.getFilename() if Utils.isDir(package_filename): # Must be a namespace package. assert Utils.python_version >= 330 package_dir = package_filename # Only include it, if it contains actual modules, which will # recurse to this one and find it again. useful = False else: package_dir = Utils.dirname(package_filename) # Real packages will always be included. useful = True debug( "Package directory %s", package_dir ) for sub_path, sub_filename in Utils.listDir(package_dir): if sub_filename in ("__init__.py", "__pycache__"): continue assert sub_path != plugin_filename if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, module.getFullName()) else: # Modules should always be included. useful = True if useful: ModuleRegistry.addRootModule(module) else: warning("Failed to include module from '%s'.", plugin_info[0])
def makeSourceDirectory(main_module): # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=R0914,R0912 assert main_module.isPythonModule() # The global context used to generate code. global_context = CodeGeneration.makeGlobalContext() # Get the full list of modules imported, create code for all of them. modules = ModuleRegistry.getDoneModules() assert main_module in modules # Sometimes we need to talk about all modules except main module. other_modules = ModuleRegistry.getDoneUserModules() # Lets check if the recurse-to modules are actually present, and warn the # user if one was not found. for any_case_module in Options.getShallFollowModules(): for module in other_modules: if module.getFullName() == any_case_module: break else: warning( "Didn't recurse to '%s', apparently not used." % \ any_case_module ) # Prepare code generation, i.e. execute finalization for it. for module in sorted(modules, key = lambda x : x.getFullName()): if module.isPythonModule(): Finalization.prepareCodeGeneration(module) # Pick filenames. source_dir = getSourceDirectoryPath(main_module) module_filenames = pickSourceFilenames( source_dir = source_dir, modules = modules ) # First pass, generate code and use constants doing so, but prepare the # final code generation only, because constants code will be added at the # end only. prepared_modules = {} for module in sorted(modules, key = lambda x : x.getFullName()): if module.isPythonModule(): cpp_filename = module_filenames[module] prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode( global_context = global_context, module = module, module_name = module.getFullName(), other_modules = other_modules if module is main_module else () ) # Main code constants need to be allocated already too. if module is main_module and not Options.shallMakeModule(): prepared_modules[cpp_filename][1].getConstantCode(0) for module in sorted(modules, key = lambda x : x.getFullName()): if module.isPythonModule(): cpp_filename = module_filenames[module] template_values, module_context = prepared_modules[cpp_filename] source_code = CodeGeneration.generateModuleCode( module_context = module_context, template_values = template_values ) # The main of an executable module gets a bit different code. if module is main_module and not Options.shallMakeModule(): source_code = CodeGeneration.generateMainCode( main_module = main_module, context = module_context, codes = source_code ) writeSourceCode( filename = cpp_filename, source_code = source_code ) if Options.isShowInclusion(): info("Included compiled module '%s'." % module.getFullName()) elif module.isPythonShlibModule(): target_filename = Utils.joinpath( getStandaloneDirectoryPath(main_module), *module.getFullName().split('.') ) if Utils.getOS() == "Windows": target_filename += ".pyd" else: target_filename += ".so" target_dir = Utils.dirname(target_filename) if not Utils.isDir(target_dir): Utils.makePath(target_dir) shutil.copy( module.getFilename(), target_filename ) standalone_entry_points.append( (target_filename, module.getPackage()) ) else: assert False, module writeSourceCode( filename = Utils.joinpath(source_dir, "__constants.cpp"), source_code = ConstantCodes.getConstantsDefinitionCode( context = global_context ) ) helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode() writeSourceCode( filename = Utils.joinpath(source_dir, "__helpers.hpp"), source_code = helper_decl_code ) writeSourceCode( filename = Utils.joinpath(source_dir, "__helpers.cpp"), source_code = helper_impl_code )
def getDependsExePath(): """ Return the path of depends.exe (for Windows). Will prompt the user to download if not already cached in AppData directory for Nuitka. """ if Utils.getArchitecture() == "x86": depends_url = "http://dependencywalker.com/depends22_x86.zip" else: depends_url = "http://dependencywalker.com/depends22_x64.zip" if "APPDATA" not in os.environ: sys.exit("Error, standalone mode cannot find 'APPDATA' environment.") nuitka_app_dir = Utils.joinpath(os.environ["APPDATA"], "nuitka") if not Utils.isDir(nuitka_app_dir): Utils.makePath(nuitka_app_dir) nuitka_depends_zip = Utils.joinpath( nuitka_app_dir, Utils.basename(depends_url) ) if not Utils.isFile(nuitka_depends_zip): Tracing.printLine("""\ Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool to analyze the dependencies of Python extension modules. Is it OK to download and put it in APPDATA (no installer needed, cached, one time question).""") reply = raw_input("Proceed and download? [Yes]/No ") if reply.lower() in ("no", 'n'): sys.exit("Nuitka does not work in --standalone on Windows without.") info("Downloading '%s'" % depends_url) urlretrieve( depends_url, nuitka_depends_zip ) nuitka_depends_dir = Utils.joinpath( nuitka_app_dir, Utils.getArchitecture() ) if not Utils.isDir(nuitka_depends_dir): os.makedirs(nuitka_depends_dir) depends_exe = os.path.join( nuitka_depends_dir, "depends.exe" ) if not Utils.isFile(depends_exe): info("Extracting to '%s'" % depends_exe) import zipfile try: depends_zip = zipfile.ZipFile(nuitka_depends_zip) depends_zip.extractall(nuitka_depends_dir) except Exception: # Catching anything zip throws, pylint:disable=W0703 info("Problem with the downloaded zip file, deleting it.") Utils.deleteFile(depends_exe, must_exist = False) Utils.deleteFile(nuitka_depends_zip, must_exist = True) sys.exit( "Error, need '%s' as extracted from '%s'." % ( depends_exe, depends_url ) ) assert Utils.isFile(depends_exe) return depends_exe
def _findModuleInPath2(module_name, search_path): """ This is out own module finding low level implementation. Just the full module name and search path are given. This is then tasked to raise "ImportError" or return a path if it finds it, or None, if it is a built-in. """ # We have many branches here, because there are a lot of cases to try. # pylint: disable=R0912 # We may have to decide between package and module, therefore build # a list of candidates. candidates = oset.OrderedSet() considered = set() for entry in search_path: # Don't try again, just with an entry of different casing or complete # duplicate. if Utils.normcase(entry) in considered: continue considered.add(Utils.normcase(entry)) package_directory = os.path.join(entry, module_name) # First, check for a package with an init file, that would be the # first choice. if Utils.isDir(package_directory): for suffix in (".py", ".pyc"): package_file_name = "__init__" + suffix file_path = os.path.join(package_directory, package_file_name) if Utils.isFile(file_path): candidates.add( (entry, 1, package_directory) ) break else: if python_version >= 330: candidates.add( (entry, 2, package_directory) ) # Then, check out suffixes of all kinds. for suffix, _mode, _type in imp.get_suffixes(): file_path = Utils.joinpath(entry, module_name + suffix) if Utils.isFile(file_path): candidates.add( (entry, 1, file_path) ) break if _debug_module_finding: print("Candidates", candidates) if candidates: # Ignore lower priority matches from package directories without # "__init__.py" file. min_prio = min(candidate[1] for candidate in candidates) candidates = [ candidate for candidate in candidates if candidate[1] == min_prio ] # On case sensitive systems, no resolution needed. if case_sensitive: return candidates[0][2] else: for candidate in candidates: dir_listing = os.listdir(candidate[0]) for filename in dir_listing: if Utils.joinpath(candidate[0], filename) == candidate[2]: return candidate[2] # Only exact case matches matter, all candidates were ignored, # lets just fall through to raising the import error. # Nothing found. raise ImportError
def makeSourceDirectory(main_module): # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=R0914,R0912 assert main_module.isPythonModule() # The global context used to generate code. global_context = CodeGeneration.makeGlobalContext() # Get the full list of modules imported, create code for all of them. modules = ModuleRegistry.getDoneModules() assert main_module in modules # Sometimes we need to talk about all modules except main module. other_modules = ModuleRegistry.getDoneUserModules() # Lets check if the recurse-to modules are actually present, and warn the # user if one was not found. for any_case_module in Options.getShallFollowModules(): for module in other_modules: if module.getFullName() == any_case_module: break else: warning( "Didn't recurse to '%s', apparently not used." % \ any_case_module ) # Prepare code generation, i.e. execute finalization for it. for module in sorted(modules, key=lambda x: x.getFullName()): if module.isPythonModule(): Finalization.prepareCodeGeneration(module) # Pick filenames. source_dir = getSourceDirectoryPath(main_module) module_filenames = pickSourceFilenames(source_dir=source_dir, modules=modules) # First pass, generate code and use constants doing so, but prepare the # final code generation only, because constants code will be added at the # end only. prepared_modules = {} for module in sorted(modules, key=lambda x: x.getFullName()): if module.isPythonModule(): cpp_filename = module_filenames[module] prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode( global_context=global_context, module=module, module_name=module.getFullName(), other_modules=other_modules if module is main_module else ()) # Main code constants need to be allocated already too. if module is main_module and not Options.shallMakeModule(): prepared_modules[cpp_filename][1].getConstantCode(0) for module in sorted(modules, key=lambda x: x.getFullName()): if module.isPythonModule(): cpp_filename = module_filenames[module] template_values, module_context = prepared_modules[cpp_filename] source_code = CodeGeneration.generateModuleCode( module_context=module_context, template_values=template_values) # The main of an executable module gets a bit different code. if module is main_module and not Options.shallMakeModule(): source_code = CodeGeneration.generateMainCode( main_module=main_module, context=module_context, codes=source_code) writeSourceCode(filename=cpp_filename, source_code=source_code) if Options.isShowInclusion(): info("Included compiled module '%s'." % module.getFullName()) elif module.isPythonShlibModule(): target_filename = Utils.joinpath( getStandaloneDirectoryPath(main_module), *module.getFullName().split('.')) if Utils.getOS() == "Windows": target_filename += ".pyd" else: target_filename += ".so" target_dir = Utils.dirname(target_filename) if not Utils.isDir(target_dir): Utils.makePath(target_dir) shutil.copy(module.getFilename(), target_filename) standalone_entry_points.append( (target_filename, module.getPackage())) else: assert False, module writeSourceCode(filename=Utils.joinpath(source_dir, "__constants.cpp"), source_code=ConstantCodes.getConstantsDefinitionCode( context=global_context)) helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode() writeSourceCode(filename=Utils.joinpath(source_dir, "__helpers.hpp"), source_code=helper_decl_code) writeSourceCode(filename=Utils.joinpath(source_dir, "__helpers.cpp"), source_code=helper_impl_code)
def getStandardLibraryPaths(): """ Get the standard library paths. """ # Using the function object to cache its result, avoiding global variable # usage. if not hasattr(getStandardLibraryPaths, "result"): os_filename = os.__file__ if os_filename.endswith(".pyc"): os_filename = os_filename[:-1] os_path = Utils.normcase(Utils.dirname(os_filename)) stdlib_paths = set([os_path]) # Happens for virtualenv situation, some modules will come from the link # this points to. if Utils.isLink(os_filename): os_filename = Utils.readLink(os_filename) stdlib_paths.add(Utils.normcase(Utils.dirname(os_filename))) # Another possibility is "orig-prefix.txt" file near the os.py, which # points to the original install. orig_prefix_filename = Utils.joinpath(os_path, "orig-prefix.txt") if Utils.isFile(orig_prefix_filename): # Scan upwards, until we find a "bin" folder, with "activate" to # locate the structural path to be added. We do not know for sure # if there is a sub-directory under "lib" to use or not. So we try # to detect it. search = os_path lib_part = "" while os.path.splitdrive(search)[1] not in (os.path.sep, ""): if Utils.isFile(Utils.joinpath(search,"bin/activate")) or \ Utils.isFile(Utils.joinpath(search,"scripts/activate")): break lib_part = Utils.joinpath(Utils.basename(search), lib_part) search = Utils.dirname(search) assert search and lib_part stdlib_paths.add( Utils.normcase( Utils.joinpath( open(orig_prefix_filename).read(), lib_part, ) ) ) # And yet another possibility, for MacOS Homebrew created virtualenv # at least is a link ".Python", which points to the original install. python_link_filename = Utils.joinpath(os_path, "..", ".Python") if Utils.isLink(python_link_filename): stdlib_paths.add( Utils.normcase( Utils.joinpath( Utils.readLink(python_link_filename), "lib" ) ) ) for stdlib_path in set(stdlib_paths): candidate = Utils.joinpath(stdlib_path, "lib-tk") if Utils.isDir(candidate): stdlib_paths.add(candidate) getStandardLibraryPaths.result = [ Utils.normcase(stdlib_path) for stdlib_path in stdlib_paths ] return getStandardLibraryPaths.result
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915 assert package is None or type(package) is str assert filename is not None if is_main and Utils.isDir(filename): source_filename = Utils.joinpath(filename, "__main__.py") if not Utils.isFile(source_filename): sys.stderr.write("%s: can't find '__main__' module in '%s'\n" % (Utils.basename(sys.argv[0]), filename)) sys.exit(2) filename = source_filename main_added = True else: main_added = False if Utils.isFile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename(filename=filename, ) if is_main: module_name = "__main__" else: module_name = Utils.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % (module_name)) sys.exit(2) if is_shlib: result = PythonShlibModule(name=module_name, package_name=package, source_ref=source_ref) elif is_main: result = PythonMainModule(main_added=main_added, source_ref=source_ref) else: if package is not None: full_name = package + "." + module_name else: full_name = module_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonModule(name=module_name, package_name=package, source_ref=source_ref) else: source_code = readSourceCodeFromFilename(module_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name=full_name, is_package=False, source_code=source_code) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name=module_name, is_package=False, bytecode=bytecode) result = UncompiledPythonModule(name=module_name, package_name=package, bytecode=bytecode, filename=filename, user_provided=True, technical=False, source_ref=source_ref) # Don't read it anymore. source_filename = None elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile(source_filename): source_ref, result = createNamespacePackage( package_name=package_name, module_relpath=filename) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename=Utils.abspath(source_filename), ) if package is not None: full_name = package + "." + package_name else: full_name = package_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonPackage(name=package_name, package_name=package, source_ref=source_ref) else: bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name=module_name, is_package=False, bytecode=bytecode) result = UncompiledPythonPackage(name=module_name, package_name=package, bytecode=bytecode, filename=filename, user_provided=True, technical=False, source_ref=source_ref) # Don't read it anymore. source_filename = None else: sys.stderr.write("%s: can't open file '%s'.\n" % (Utils.basename(sys.argv[0]), filename)) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def _consider(self, constraint_collection, module_filename, module_package): assert module_package is None or \ (type(module_package) is str and module_package != "") module_filename = Utils.normpath(module_filename) if Utils.isDir(module_filename): module_name = Utils.basename(module_filename) module_kind = "py" elif module_filename.endswith(".py"): module_name = Utils.basename(module_filename)[:-3] module_kind = "py" elif module_filename.endswith(".so"): module_kind = "shlib" module_name = Utils.basename(module_filename)[:-3] elif module_filename.endswith(".pyd"): module_kind = "shlib" module_name = Utils.basename(module_filename)[:-4] else: module_kind = None module_name = None if module_kind is not None: decision, reason = decideRecursion(module_filename=module_filename, module_name=module_name, module_package=module_package, module_kind=module_kind) if decision: module_relpath = Utils.relpath(module_filename) imported_module, added_flag = recurseTo( module_package=module_package, module_filename=module_filename, module_relpath=module_relpath, module_kind=module_kind, reason=reason) if added_flag: constraint_collection.signalChange( "new_code", imported_module.getSourceReference(), "Recursed to module.") return imported_module elif decision is None and module_kind == "py": if module_package is None: module_fullpath = module_name else: module_fullpath = module_package + '.' + module_name if module_filename not in self._warned_about and \ module_fullpath not in getModuleWhiteList(): self._warned_about.add(module_filename) warning("""\ Not recursing to '%(full_path)s' (%(filename)s), please specify \ --recurse-none (do not warn), \ --recurse-all (recurse to all), \ --recurse-not-to=%(full_path)s (ignore it), \ --recurse-to=%(full_path)s (recurse to it) to change.""" % { "full_path": module_fullpath, "filename": module_filename })
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912,R0915 assert package is None or type(package) is str assert filename is not None if is_main and Utils.isDir(filename): source_filename = Utils.joinpath(filename, "__main__.py") if not Utils.isFile(source_filename): sys.stderr.write( "%s: can't find '__main__' module in '%s'\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) filename = source_filename main_added = True else: main_added = False if Utils.isFile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename( filename = filename, ) if is_main: module_name = "__main__" else: module_name = Utils.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % ( module_name ) ) sys.exit(2) if is_shlib: result = PythonShlibModule( name = module_name, package_name = package, source_ref = source_ref ) elif is_main: result = PythonMainModule( main_added = main_added, source_ref = source_ref ) else: if package is not None: full_name = package + "." + module_name else: full_name = module_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonModule( name = module_name, package_name = package, source_ref = source_ref ) else: source_code = readSourceCodeFromFilename(module_name, filename) source_code = Plugins.onFrozenModuleSourceCode( module_name = full_name, is_package = False, source_code = source_code ) bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = module_name, is_package = False, bytecode = bytecode ) result = UncompiledPythonModule( name = module_name, package_name = package, bytecode = bytecode, filename = filename, user_provided = True, technical = False, source_ref = source_ref ) # Don't read it anymore. source_filename = None elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile(source_filename): source_ref, result = createNamespacePackage( package_name = package_name, module_relpath = filename ) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename = Utils.abspath(source_filename), ) if package is not None: full_name = package + "." + package_name else: full_name = package_name decision = Plugins.decideCompilation(full_name, source_ref) if decision == "compiled": result = CompiledPythonPackage( name = package_name, package_name = package, source_ref = source_ref ) else: bytecode = compile(source_code, filename, "exec") bytecode = Plugins.onFrozenModuleBytecode( module_name = module_name, is_package = False, bytecode = bytecode ) result = UncompiledPythonPackage( name = module_name, package_name = package, bytecode = bytecode, filename = filename, user_provided = True, technical = False, source_ref = source_ref ) # Don't read it anymore. source_filename = None else: sys.stderr.write( "%s: can't open file '%s'.\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def _checkPluginPath(plugin_filename, module_package): # Many branches, for the decision is very complex debug( "Checking detail plug-in path '%s' '%s':", plugin_filename, module_package ) plugin_info = considerFilename( module_filename = plugin_filename ) if plugin_info is not None: module, is_added = recurseTo( module_filename = plugin_info[0], module_relpath = plugin_info[1], module_package = module_package, module_kind = "py", reason = "Lives in plug-in directory." ) if module: if not is_added: warning( "Recursed to %s '%s' at '%s' twice.", "package" if module.isCompiledPythonPackage() else "module", module.getName(), plugin_info[0] ) if not isSameModulePath(module.getFilename(), plugin_info[0]): warning( "Duplicate ignored '%s'.", plugin_info[1] ) return debug( "Recursed to %s %s %s", module.getName(), module.getPackage(), module ) ImportCache.addImportedModule(module) if module.isCompiledPythonPackage(): package_filename = module.getFilename() if Utils.isDir(package_filename): # Must be a namespace package. assert python_version >= 330 package_dir = package_filename # Only include it, if it contains actual modules, which will # recurse to this one and find it again. else: package_dir = Utils.dirname(package_filename) # Real packages will always be included. ModuleRegistry.addRootModule(module) debug( "Package directory %s", package_dir ) for sub_path, sub_filename in Utils.listDir(package_dir): if sub_filename in ("__init__.py", "__pycache__"): continue assert sub_path != plugin_filename if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, module.getFullName()) elif module.isCompiledPythonModule(): ModuleRegistry.addRootModule(module) else: warning("Failed to include module from '%s'.", plugin_info[0])
def _findModuleInPath2(module_name, search_path): """ This is out own module finding low level implementation. Just the full module name and search path are given. This is then tasked to raise "ImportError" or return a path if it finds it, or None, if it is a built-in. """ # We have many branches here, because there are a lot of cases to try. # pylint: disable=R0912 # We may have to decide between package and module, therefore build # a list of candidates. candidates = oset.OrderedSet() considered = set() for entry in search_path: # Don't try again, just with an entry of different casing or complete # duplicate. if Utils.normcase(entry) in considered: continue considered.add(Utils.normcase(entry)) package_directory = os.path.join(entry, module_name) # First, check for a package with an init file, that would be the # first choice. if Utils.isDir(package_directory): for suffix in (".py", ".pyc"): package_file_name = "__init__" + suffix file_path = os.path.join(package_directory, package_file_name) if Utils.isFile(file_path): candidates.add( (entry, 1, package_directory) ) break else: if Utils.python_version >= 330: candidates.add( (entry, 2, package_directory) ) # Then, check out suffixes of all kinds. for suffix, _mode, _type in imp.get_suffixes(): file_path = Utils.joinpath(entry, module_name + suffix) if Utils.isFile(file_path): candidates.add( (entry, 1, file_path) ) break if _debug_module_finding: print("Candidates", candidates) if candidates: # Ignore lower priority matches from package directories without # "__init__.py" file. min_prio = min(candidate[1] for candidate in candidates) candidates = [ candidate for candidate in candidates if candidate[1] == min_prio ] # On case sensitive systems, no resolution needed. if case_sensitive: return candidates[0][2] else: for candidate in candidates: dir_listing = os.listdir(candidate[0]) for filename in dir_listing: if Utils.joinpath(candidate[0], filename) == candidate[2]: return candidate[2] # Only excact case matches matter, all candidates were ignored, # lets just fall through to raising the import error. # Nothing found. raise ImportError
def decideModuleTree(filename, package, is_shlib, is_top, is_main): # Many variables, branches, due to the many cases, pylint: disable=R0912 assert package is None or type(package) is str assert filename is not None if is_main and Utils.isDir(filename): source_filename = Utils.joinpath(filename, "__main__.py") if not Utils.isFile(source_filename): sys.stderr.write( "%s: can't find '__main__' module in '%s'\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) filename = source_filename main_added = True else: main_added = False if Utils.isFile(filename): source_filename = filename source_ref = SourceCodeReferences.fromFilename( filename = filename, ) if is_main: module_name = "__main__" else: module_name = Utils.basename(filename) if module_name.endswith(".py"): module_name = module_name[:-3] if is_shlib: module_name = module_name.split('.')[0] if '.' in module_name: sys.stderr.write( "Error, '%s' is not a proper python module name.\n" % ( module_name ) ) sys.exit(2) if is_shlib: result = PythonShlibModule( name = module_name, package_name = package, source_ref = source_ref ) elif is_main: result = PythonMainModule( main_added = main_added, source_ref = source_ref ) else: result = PythonModule( name = module_name, package_name = package, source_ref = source_ref ) elif Importing.isPackageDir(filename): if is_top: package_name = Utils.splitpath(filename)[-1] else: package_name = Utils.basename(filename) source_filename = Utils.joinpath(filename, "__init__.py") if not Utils.isFile(source_filename): source_ref, result = createNamespacePackage( package_name = package_name, module_relpath = filename ) source_filename = None else: source_ref = SourceCodeReferences.fromFilename( filename = Utils.abspath(source_filename), ) result = PythonPackage( name = package_name, package_name = package, source_ref = source_ref ) else: sys.stderr.write( "%s: can't open file '%s'.\n" % ( Utils.basename(sys.argv[0]), filename ) ) sys.exit(2) if not Options.shallHaveStatementLines(): source_ref = source_ref.atInternal() return result, source_ref, source_filename
def _consider(self, constraint_collection, module_filename, module_package): assert module_package is None or \ (type(module_package) is str and module_package != "") module_filename = Utils.normpath(module_filename) if Utils.isDir(module_filename): module_name = Utils.basename(module_filename) module_kind = "py" elif module_filename.endswith(".py"): module_name = Utils.basename(module_filename)[:-3] module_kind = "py" elif module_filename.endswith(".so"): module_kind = "shlib" module_name = Utils.basename(module_filename)[:-3] elif module_filename.endswith(".pyd"): module_kind = "shlib" module_name = Utils.basename(module_filename)[:-4] else: module_kind = None module_name = None if module_kind is not None: decision, reason = decideRecursion( module_filename = module_filename, module_name = module_name, module_package = module_package, module_kind = module_kind ) if decision: module_relpath = Utils.relpath(module_filename) imported_module, added_flag = recurseTo( module_package = module_package, module_filename = module_filename, module_relpath = module_relpath, module_kind = module_kind, reason = reason ) if added_flag: constraint_collection.signalChange( "new_code", imported_module.getSourceReference(), "Recursed to module." ) return imported_module elif decision is None and module_kind == "py": if module_package is None: module_fullpath = module_name else: module_fullpath = module_package + '.' + module_name if module_filename not in self._warned_about and \ module_fullpath not in getModuleWhiteList(): self._warned_about.add(module_filename) warning( """\ Not recursing to '%(full_path)s' (%(filename)s), please specify \ --recurse-none (do not warn), \ --recurse-all (recurse to all), \ --recurse-not-to=%(full_path)s (ignore it), \ --recurse-to=%(full_path)s (recurse to it) to change.""" % { "full_path" : module_fullpath, "filename" : module_filename } )
def makeSourceDirectory(main_module): """ Get the full list of modules imported, create code for all of them. """ # We deal with a lot of details here, but rather one by one, and split makes # no sense, pylint: disable=R0912,R0914 assert main_module.isCompiledPythonModule() # The global context used to generate code. global_context = CodeGeneration.makeGlobalContext() assert main_module in ModuleRegistry.getDoneModules() # We might have chosen to include it as bytecode, and only compiled it for # fun, and to find its imports. In this case, now we just can drop it. Or # a module may shadow a frozen module, but be a different one, then we can # drop the frozen one. # TODO: This really should be done when the compiled module comes into # existence. for module in ModuleRegistry.getDoneUserModules(): if module.isCompiledPythonModule(): uncompiled_module = ModuleRegistry.getUncompiledModule( module_name = module.getFullName(), module_filename = module.getCompileTimeFilename() ) if uncompiled_module is not None: # We now need to decide which one to keep, compiled or uncompiled # module. Some uncompiled modules may have been asked by the user # or technically required. By default, frozen code if it exists # is preferred, as it will be from standalone mode adding it. if uncompiled_module.isUserProvided(): ModuleRegistry.removeDoneModule(module) else: ModuleRegistry.removeUncompiledModule(uncompiled_module) # Lets check if the recurse-to modules are actually present, and warn the # user if one of those was not found. for any_case_module in Options.getShallFollowModules(): for module in ModuleRegistry.getDoneUserModules(): if module.getFullName() == any_case_module: break else: warning( "Didn't recurse to '%s', apparently not used." % \ any_case_module ) # Prepare code generation, i.e. execute finalization for it. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): Finalization.prepareCodeGeneration(module) # Pick filenames. source_dir = getSourceDirectoryPath(main_module) module_filenames = pickSourceFilenames( source_dir = source_dir, modules = ModuleRegistry.getDoneModules() ) # First pass, generate code and use constants doing so, but prepare the # final code generation only, because constants code will be added at the # end only. prepared_modules = {} for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): cpp_filename = module_filenames[module] prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode( global_context = global_context, module = module, module_name = module.getFullName(), ) # Main code constants need to be allocated already too. if module is main_module and not Options.shallMakeModule(): prepared_modules[cpp_filename][1].getConstantCode(0) # Second pass, generate the actual module code into the files. for module in ModuleRegistry.getDoneModules(): if module.isCompiledPythonModule(): cpp_filename = module_filenames[module] template_values, module_context = prepared_modules[cpp_filename] source_code = CodeGeneration.generateModuleCode( module_context = module_context, template_values = template_values ) # The main of an executable module gets a bit different code. if module is main_module and not Options.shallMakeModule(): source_code = MainCodes.generateMainCode( main_module = main_module, context = module_context, codes = source_code ) writeSourceCode( filename = cpp_filename, source_code = source_code ) if Options.isShowInclusion(): info("Included compiled module '%s'." % module.getFullName()) elif module.isPythonShlibModule(): target_filename = Utils.joinpath( getStandaloneDirectoryPath(main_module), *module.getFullName().split('.') ) if Utils.getOS() == "Windows": target_filename += ".pyd" else: target_filename += ".so" target_dir = Utils.dirname(target_filename) if not Utils.isDir(target_dir): Utils.makePath(target_dir) shutil.copy( module.getFilename(), target_filename ) standalone_entry_points.append( (target_filename, module.getPackage()) ) else: assert False, module writeSourceCode( filename = Utils.joinpath(source_dir, "__constants.cpp"), source_code = ConstantCodes.getConstantsDefinitionCode( context = global_context ) ) helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode( ModuleRegistry.getDoneUserModules() ) writeSourceCode( filename = Utils.joinpath(source_dir, "__helpers.hpp"), source_code = helper_decl_code ) writeSourceCode( filename = Utils.joinpath(source_dir, "__helpers.cpp"), source_code = helper_impl_code )
def getDependsExePath(): """ Return the path of depends.exe (for Windows). Will prompt the user to download if not already cached in AppData directory for Nuitka. """ if Utils.getArchitecture() == "x86": depends_url = "http://dependencywalker.com/depends22_x86.zip" else: depends_url = "http://dependencywalker.com/depends22_x64.zip" if "APPDATA" not in os.environ: sys.exit("Error, standalone mode cannot find 'APPDATA' environment.") nuitka_app_dir = Utils.joinpath(os.environ["APPDATA"], "nuitka") if not Utils.isDir(nuitka_app_dir): Utils.makePath(nuitka_app_dir) nuitka_depends_zip = Utils.joinpath(nuitka_app_dir, Utils.basename(depends_url)) if not Utils.isFile(nuitka_depends_zip): Tracing.printLine("""\ Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool to analyze the dependencies of Python extension modules. Is it OK to download and put it in APPDATA (no installer needed, cached, one time question).""") reply = raw_input("Proceed and download? [Yes]/No ") if reply.lower() in ("no", 'n'): sys.exit( "Nuitka does not work in --standalone on Windows without.") info("Downloading '%s'" % depends_url) urlretrieve(depends_url, nuitka_depends_zip) nuitka_depends_dir = Utils.joinpath(nuitka_app_dir, Utils.getArchitecture()) if not Utils.isDir(nuitka_depends_dir): os.makedirs(nuitka_depends_dir) depends_exe = os.path.join(nuitka_depends_dir, "depends.exe") if not Utils.isFile(depends_exe): info("Extracting to '%s'" % depends_exe) import zipfile try: depends_zip = zipfile.ZipFile(nuitka_depends_zip) depends_zip.extractall(nuitka_depends_dir) except Exception: # Catching anything zip throws, pylint:disable=W0703 info("Problem with the downloaded zip file, deleting it.") Utils.deleteFile(depends_exe, must_exist=False) Utils.deleteFile(nuitka_depends_zip, must_exist=True) sys.exit("Error, need '%s' as extracted from '%s'." % (depends_exe, depends_url)) assert Utils.isFile(depends_exe) return depends_exe
def getStandardLibraryPaths(): """ Get the standard library paths. """ # Using the function object to cache its result, avoiding global variable # usage. if not hasattr(getStandardLibraryPaths, "result"): os_filename = os.__file__ if os_filename.endswith(".pyc"): os_filename = os_filename[:-1] os_path = Utils.normcase(Utils.dirname(os_filename)) stdlib_paths = set([os_path]) # Happens for virtualenv situation, some modules will come from the link # this points to. if Utils.isLink(os_filename): os_filename = Utils.readLink(os_filename) stdlib_paths.add(Utils.normcase(Utils.dirname(os_filename))) # Another possibility is "orig-prefix.txt" file near the os.py, which # points to the original install. orig_prefix_filename = Utils.joinpath(os_path, "orig-prefix.txt") if Utils.isFile(orig_prefix_filename): # Scan upwards, until we find a "bin" folder, with "activate" to # locate the structural path to be added. We do not know for sure # if there is a sub-directory under "lib" to use or not. So we try # to detect it. search = os_path lib_part = "" while os.path.splitdrive(search)[1] not in (os.path.sep, ""): if Utils.isFile(Utils.joinpath(search,"bin/activate")) or \ Utils.isFile(Utils.joinpath(search,"scripts/activate")): break lib_part = Utils.joinpath(Utils.basename(search), lib_part) search = Utils.dirname(search) assert search and lib_part stdlib_paths.add( Utils.normcase( Utils.joinpath( open(orig_prefix_filename).read(), lib_part, ))) # And yet another possibility, for MacOS Homebrew created virtualenv # at least is a link ".Python", which points to the original install. python_link_filename = Utils.joinpath(os_path, "..", ".Python") if Utils.isLink(python_link_filename): stdlib_paths.add( Utils.normcase( Utils.joinpath(Utils.readLink(python_link_filename), "lib"))) for stdlib_path in set(stdlib_paths): candidate = Utils.joinpath(stdlib_path, "lib-tk") if Utils.isDir(candidate): stdlib_paths.add(candidate) getStandardLibraryPaths.result = [ Utils.normcase(stdlib_path) for stdlib_path in stdlib_paths ] return getStandardLibraryPaths.result