def _checkPluginPath(plugin_filename, module_package): plugin_info = considerFilename(module_package=module_package, module_filename=plugin_filename) if plugin_info is not None: module, added = recurseTo(module_filename=plugin_info[0], module_relpath=plugin_info[1], module_package=module_package) if module: if not added: warning("Recursed to %s '%s' at '%s' twice.", "package" if module.isPythonPackage() else "module", module.getName(), plugin_info[0]) if module.isPythonPackage(): package_dir = Utils.dirname(module.getFilename()) for sub_path, sub_filename in Utils.listDir(package_dir): if sub_filename == "__init__.py": continue assert sub_path != plugin_filename, package_dir if Importing.isPackageDir(sub_path) or sub_path.endswith( ".py"): _checkPluginPath(sub_path, module.getFullName()) else: warning("Failed to include module from '%s'.", plugin_info[0])
def runScons( options, quiet ): # For the scons file to find the static C++ files and include path. The scons file is # unable to use __file__ for the task. os.environ[ "NUITKA_SCONS" ] = getSconsDataPath() if os.name == "nt": # On Windows this Scons variable must be set by us. os.environ[ "SCONS_LIB_DIR" ] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.0.1" ) # Also, for MinGW we can avoid the user having to add the path if he used the # default path or installed it on the same drive by appending to the PATH variable # before executing scons. os.environ[ "PATH" ] += r";\MinGW\bin;C:\MinGW\bin" scons_command = """%(python)s %(binary)s %(quiet)s -f %(scons_file)s --jobs %(job_limit)d %(options)s""" % { "python" : sys.executable if Utils.python_version < 300 else "python", "binary" : getSconsBinaryPath(), "quiet" : "--quiet" if quiet else "", "scons_file" : Utils.joinpath( getSconsDataPath(), "SingleExe.scons" ), "job_limit" : Options.getJobLimit(), "options" : " ".join( "%s=%s" % ( key, value ) for key, value in options.items() ) } if Options.isShowScons(): Tracing.printLine( "Scons command:", scons_command ) return 0 == os.system( scons_command )
def checkPluginFilenamePattern(pattern): import sys, glob debug( "Checking plug-in pattern '%s':", pattern, ) if Utils.isDir(pattern): sys.exit("Error, pattern cannot be a directory name.") found = False for filename in glob.iglob(pattern): if filename.endswith(".pyc"): continue if not Utils.isFile(filename): continue found = True _checkPluginPath(filename, None) if not found: warning("Didn't match any files against pattern '%s'." % pattern)
def considerImplicitImports(self, signal_change): for module_name, module_package in self.getImplicitImports(): _module_package, _module_name, module_filename = \ Importing.findModule( source_ref = self.source_ref, module_name = module_name, parent_package = module_package, level = -1, warn = True ) if module_filename is None: sys.exit( "Error, implicit module '%s' expected by '%s' not found" % ( module_name, self.getFullName() ) ) elif Utils.isDir(module_filename): module_kind = "py" elif module_filename.endswith(".py"): module_kind = "py" elif module_filename.endswith(".so"): module_kind = "shlib" elif module_filename.endswith(".pyd"): module_kind = "shlib" else: assert False, module_filename from nuitka.tree import Recursion decision, reason = Recursion.decideRecursion( module_filename = module_filename, module_name = module_name, module_package = module_package, module_kind = module_kind ) assert decision or reason == "Module is frozen." if decision: module_relpath = Utils.relpath(module_filename) imported_module, added_flag = Recursion.recurseTo( module_package = module_package, module_filename = module_filename, module_relpath = module_relpath, module_kind = module_kind, reason = reason ) from nuitka.ModuleRegistry import addUsedModule addUsedModule(imported_module) if added_flag: signal_change( "new_code", imported_module.getSourceReference(), "Recursed to module." )
def _detectedSourceFile(filename, module_name, result, is_late): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile(filename=filename, module_name="_collections_abc", result=result, is_late=is_late) source_code = readSourceCodeFromFilename(filename) if Utils.python_version >= 300: filename = filename.decode("utf-8") if module_name == "site": source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, Utils.basename(filename), source_code) debug("Freezing module '%s' (from '%s').", module_name, filename) result.append( (module_name, marshal.dumps(compile(source_code, filename, "exec")), Utils.basename(filename) == "__init__.py", filename, is_late)) module_names.add(module_name)
def checkPluginPath(plugin_filename, module_package): debug( "Checking top level plugin path %s %s", plugin_filename, module_package ) plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: # File or package makes a difference, handle that if Utils.isFile(plugin_info[0]) or \ Importing.isPackageDir(plugin_info[0]): _checkPluginPath(plugin_filename, module_package) elif Utils.isDir(plugin_info[0]): for sub_path, sub_filename in Utils.listDir(plugin_info[0]): assert sub_filename != "__init__.py" if Importing.isPackageDir(sub_path) or \ sub_path.endswith(".py"): _checkPluginPath(sub_path, None) else: warning("Failed to include module from '%s'.", plugin_info[0]) else: warning("Failed to recurse to directory '%s'.", plugin_filename)
def detectBinaryDLLs(binary_filename, package_name): """ Detect the DLLs used by a binary. Using ldd (Linux), depends.exe (Windows), or otool (MacOS) the list of used DLLs is retrieved. """ if Utils.getOS() in ("Linux", "NetBSD"): # TODO: FreeBSD may work the same way, not tested. return _detectBinaryPathDLLsLinuxBSD( binary_filename = binary_filename ) elif Utils.getOS() == "Windows": return _detectBinaryPathDLLsWindows( binary_filename = binary_filename, package_name = package_name ) elif Utils.getOS() == "Darwin": return _detectBinaryPathDLLsLinuxBSD( binary_filename = binary_filename ) else: # Support your platform above. assert False, Utils.getOS()
def getSconsBinaryCall(): """ Return a way to execute Scons. Using potentially inline copy if no system Scons is available or if we are on Windows. """ if Utils.isFile("/usr/bin/scons"): return ["/usr/bin/scons"] else: return [getPython2ExePath(), Utils.joinpath(getSconsInlinePath(), "bin", "scons.py")]
def getVariableCodeName(in_context, variable): if in_context: # Closure case: return "closure_" + Utils.encodeNonAscii(variable.getName()) elif variable.isParameterVariable(): return "par_" + Utils.encodeNonAscii(variable.getName()) elif variable.isTempVariable(): return "tmp_" + Utils.encodeNonAscii(variable.getName()) else: return "var_" + Utils.encodeNonAscii(variable.getName())
def getSconsBinaryCall(): """ Return a way to execute Scons. Using potentially inline copy if no system Scons is available or if we are on Windows. """ if Utils.isFile("/usr/bin/scons"): return ["/usr/bin/scons"] else: return [ getPython2ExePath(), Utils.joinpath(getSconsInlinePath(), "bin", "scons.py") ]
def __init__(self, *args): QtGui.QDialog.__init__(self, *args) ui_dir = Utils.dirname(__file__) ui_filename = Utils.joinpath(ui_dir, "dialogs", "InspectPythonTree.ui") uic.loadUi(ui_filename, self) self.treeview_nodes.setSelectionMode(self.treeview_nodes.SingleSelection) self.displayed = None self.source_code = None self.model = None self.moving = None
def copyUsedDLLs(dist_dir, binary_filename, standalone_entry_points): dll_map = [] used_dlls = detectUsedDLLs(standalone_entry_points) for dll_filename1, sources1 in iterItems(used_dlls): for dll_filename2, sources2 in iterItems(used_dlls): if dll_filename1 == dll_filename2: continue # Colliding basenames are an issue to us. if Utils.basename(dll_filename1) != Utils.basename(dll_filename2): continue dll_name = Utils.basename(dll_filename1) if Options.isShowInclusion(): info("""Colliding DLL names for %s, checking identity of \ '%s' <-> '%s'.""" % ( dll_name, dll_filename1, dll_filename2, )) # Check that if a DLL has the same name, if it's identical, # happens at least for OSC and Fedora 20. import filecmp if filecmp.cmp(dll_filename1, dll_filename2): continue sys.exit("""Error, conflicting DLLs for '%s' \ (%s used by %s different from %s used by %s).""" % (dll_name, dll_filename1, ", ".join(sources1), dll_filename2, ", ".join(sources2))) for dll_filename, sources in iterItems(used_dlls): dll_name = Utils.basename(dll_filename) target_path = Utils.joinpath(dist_dir, dll_name) shutil.copy(dll_filename, target_path) dll_map.append((dll_filename, dll_name)) if Options.isShowInclusion(): info("Included used shared library '%s' (used by %s)." % (dll_filename, ", ".join(sources))) if Utils.getOS() == "Darwin": # For MacOS, the binary needs to be changed to reflect the DLL # location in the dist folder. fixupBinaryDLLPaths(binary_filename, dll_map) if Utils.getOS() == "Linux": # For Linux, the rpath of libraries may be an issue. for _original_path, dll_filename in dll_map: removeSharedLibraryRPATH(Utils.joinpath(dist_dir, dll_filename)) for standalone_entry_point in standalone_entry_points[1:]: removeSharedLibraryRPATH(standalone_entry_point[0])
def __init__(self, *args): QtGui.QDialog.__init__( self, *args ) ui_dir = Utils.dirname( __file__ ) ui_filename = Utils.joinpath( ui_dir, "dialogs", "InspectPythonTree.ui" ) uic.loadUi( ui_filename, self ) self.treeview_nodes.setSelectionMode( self.treeview_nodes.SingleSelection ) self.displayed = None self.source_code = None self.model = None self.moving = None
def runScons(options, quiet): # For the scons file to find the static C++ files and include path. The # scons file is unable to use __file__ for the task. os.environ["NUITKA_SCONS"] = getSconsDataPath() if Utils.getOS() == "Windows": # On Windows this Scons variable must be set by us. os.environ["SCONS_LIB_DIR"] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.3.0" ) # Also, for MinGW we can avoid the user having to add the path if he # used the default path or installed it on the same drive by appending # to the PATH variable before executing scons. os.environ["PATH"] += r";\MinGW\bin;C:\MinGW\bin" scons_command = getSconsBinaryCall() if quiet: scons_command.append("--quiet") scons_command += [ # The scons file "-f", Utils.joinpath(getSconsDataPath(), "SingleExe.scons"), # Parallel compilation. "--jobs", str(Options.getJobLimit()), # Do not warn about deprecations of Scons "--warn=no-deprecated", # Don't load "site_scons" at all. "--no-site-dir", ] if Options.isShowScons(): scons_command.append("--debug=explain") # Option values to provide to scons. for key, value in options.items(): scons_command += [key + "=" + value] if Options.isShowScons(): Tracing.printLine("Scons command:", " ".join(scons_command)) return 0 == subprocess.call(scons_command)
def runScons(options, quiet): # For the scons file to find the static C++ files and include path. The scons file is # unable to use __file__ for the task. os.environ["NUITKA_SCONS"] = getSconsDataPath() if os.name == "nt": # On Windows this Scons variable must be set by us. os.environ["SCONS_LIB_DIR"] = Utils.joinpath(getSconsInlinePath(), "lib", "scons-2.0.1") # Also, for MinGW we can avoid the user having to add the path if he used the # default path or installed it on the same drive by appending to the PATH variable # before executing scons. os.environ["PATH"] += r";\MinGW\bin;C:\MinGW\bin" # Scons is Python2 only, so we need to make the system find a suitable Python binary. if Utils.python_version < 300: python_exe = sys.executable elif os.name == "nt": if os.path.exists(r"c:\Python27\python.exe"): python_exe = r"c:\Python27\python.exe" elif os.path.exists(r"c:\Python26\python.exe"): python_exe = r"c:\Python26\python.exe" else: sys.exit( """Error, need to find Python2 executable under C:\\Python26 or \ C:\\Python27 to execute scons which is not Python3 compatible.""") else: python_exe = "python" scons_command = """%(python)s %(binary)s %(quiet)s --warn=no-deprecated -f %(scons_file)s --jobs %(job_limit)d %(options)s""" % { "python": python_exe, "binary": getSconsBinaryPath(), "quiet": "--quiet" if quiet else "", "scons_file": Utils.joinpath(getSconsDataPath(), "SingleExe.scons"), "job_limit": Options.getJobLimit(), "options": " ".join("%s=%s" % (key, value) for key, value in options.items()) } if Options.isShowScons(): Tracing.printLine("Scons command:", scons_command) return 0 == os.system(scons_command)
def buildModuleTree(filename, package, is_top, is_main): module, source_ref, source_filename = decideModuleTree( filename = filename, package = package, is_top = is_top, is_main = is_main, is_shlib = False ) addImportedModule( module_relpath = Utils.relpath(filename), imported_module = module ) # If there is source code associated (not the case for namespace packages of # Python3.3 or higher, then read it. if source_filename is not None: createModuleTree( module = module, source_ref = source_ref, source_filename = source_filename, is_main = is_main ) return module
def getRunTimeFilename(self): if Options.isStandaloneMode(): filename = self.getCompileTimeFilename() full_name = self.getFullName() result = Utils.basename(filename) current = filename for _i in range(full_name.count('.')): current = Utils.dirname(current) result = Utils.joinpath(Utils.basename(current), result) return result else: return self.getCompileTimeFilename()
def createModuleTree(module, source_ref, source_filename, is_main): if Options.isShowProgress(): memory_watch = Utils.MemoryWatch() source_code = readSourceCodeFromFilename(source_filename) module_body = buildParseTree( provider = module, source_code = source_code, source_ref = source_ref, is_module = True, is_main = is_main ) module.setBody( module_body ) completeVariableClosures(module) if Options.isShowProgress(): memory_watch.finish() Tracing.printLine( "Memory usage changed loading module '%s': %s" % ( module.getFullName(), memory_watch.asStr() ) )
def detectEarlyImports(): if Options.freezeAllStdlib(): stdlib_modules = set() # Scan the standard library paths (multiple in case of virtualenv. for stdlib_dir in getStandardLibraryPaths(): for module_name in scanStandardLibraryPath(stdlib_dir): stdlib_modules.add(module_name) import_code = 'imports = ' + repr(sorted(stdlib_modules)) + '\n'\ 'for imp in imports:\n' \ ' try:\n' \ ' __import__(imp)\n' \ ' except ImportError:\n' \ ' pass\n' else: # TODO: Should recursively include all of encodings module. import_code = "import encodings.utf_8;import encodings.ascii;import encodings.idna;" if Utils.getOS() == "Windows": import_code += "import encodings.mbcs;import encodings.cp437;" # String method hex depends on it. if Utils.python_version < 300: import_code += "import encodings.hex_codec;" import_code += "import locale;" result = _detectImports(import_code, False) debug("Finished detecting early imports.") return result
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name=module.getFullName())) global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() _optimizeModulePass(module=module, tag_set=tag_set) if not tag_set: break touched = True if _progress: memory_watch.finish() printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr())) return touched
def removeSharedLibraryRPATH(filename): process = subprocess.Popen(["readelf", "-d", filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) stdout, _stderr = process.communicate() retcode = process.poll() assert retcode == 0, filename for line in stdout.split(b"\n"): if b"RPATH" in line: if Options.isShowInclusion(): info("Removing 'RPATH' setting from '%s'.", filename) if not Utils.isExecutableCommand("chrpath"): sys.exit("""\ Error, needs 'chrpath' on your system, due to 'RPATH' settings in used shared libraries that need to be removed.""") process = subprocess.Popen(["chrpath", "-d", filename], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) process.communicate() retcode = process.poll() assert retcode == 0, filename
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine( """\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name = current_module.getFullName(), remaining = ModuleRegistry.remainingCount(), memory = Utils.getHumanReadableProcessMemoryUsage() ) ) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False if finished: break
def removeSharedLibraryRPATH(filename): process = subprocess.Popen( ["readelf", "-d", filename], stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = False ) stdout, _stderr = process.communicate() retcode = process.poll() assert retcode == 0, filename for line in stdout.split(b"\n"): if b"RPATH" in line: if Options.isShowInclusion(): info("Removing 'RPATH' setting from '%s'.", filename) if not Utils.isExecutableCommand("chrpath"): sys.exit( """\ Error, needs 'chrpath' on your system, due to 'RPATH' settings in used shared libraries that need to be removed.""" ) process = subprocess.Popen( ["chrpath", "-d", filename], stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = False ) process.communicate() retcode = process.poll() assert retcode == 0, filename
def optimize(): while True: finished = True ModuleRegistry.startTraversal() while True: current_module = ModuleRegistry.nextModule() if current_module is None: break if _progress: printLine("""\ Optimizing module '{module_name}', {remaining:d} more modules to go \ after that. Memory usage {memory}:""".format( module_name=current_module.getFullName(), remaining=ModuleRegistry.remainingCount(), memory=Utils.getHumanReadableProcessMemoryUsage())) if current_module.isPythonShlibModule(): optimizeShlibModule(current_module) else: changed = optimizePythonModule(current_module) if changed: finished = False for current_module in ModuleRegistry.getDoneModules(): if not current_module.isPythonShlibModule(): optimizeVariables(current_module) if finished: break
def attemptRecursion( self, module ): if not Options.shallMakeModule(): # Make sure the package is recursed to. module_package = module.getPackage() if module_package is not None: package_package, _package_module_name, package_filename = Importing.findModule( source_ref = module.getSourceReference(), module_name = module_package, parent_package = None, level = 1 ) imported_module, added_flag = Recursion.recurseTo( module_package = package_package, module_filename = package_filename, module_relpath = Utils.relpath( package_filename ) ) if added_flag: self.signalChange( "new_code", imported_module.getSourceReference(), "Recursed to module package." )
def optimizePythonModule(module): if _progress: printLine( "Doing module local optimizations for '{module_name}'.".format( module_name=module.getFullName())) # The tag set is global, so it can react to changes without context. # pylint: disable=W0603 global tag_set tag_set = TagSet() touched = False if _progress: memory_watch = Utils.MemoryWatch() while True: tag_set.clear() _optimizeModulePass(module=module) if not tag_set: break touched = True if _progress: memory_watch.finish() printLine("Memory usage changed during optimization of '%s': %s" % (module.getFullName(), memory_watch.asStr())) return touched
def _getPython2ExePathWindows(): # Shortcuts for the default installation directories, to avoid going to # registry at all unless necessary. Any Python2 will do for Scons, so it # can be avoided. if os.path.isfile(r"c:\Python27\python.exe"): return r"c:\Python27\python.exe" elif os.path.isfile(r"c:\Python26\python.exe"): return r"c:\Python26\python.exe" # Windows only code, pylint: disable=E0602,F0401,I0021 try: import _winreg as winreg except ImportError: import winreg # lint:ok for search in ("2.7", "2.6"): for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY: try: key = winreg.OpenKey( winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search, 0, winreg.KEY_READ | arch_key) return Utils.joinpath(winreg.QueryValue(key, ""), "python.exe") except WindowsError: # lint:ok pass
def _getPython2ExePathWindows(): # Shortcuts for the default installation directories, to avoid going to # registry at all. if os.path.isfile(r"c:\Python27\python.exe"): return r"c:\Python27\python.exe" elif os.path.isfile(r"c:\Python26\python.exe"): return r"c:\Python26\python.exe" # Windows only code, pylint: disable=E0602,F0401 try: import _winreg as winreg except ImportError: import winreg # lint:ok for search in ("2.7", "2.6"): for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY: try: key = winreg.OpenKey( winreg.HKEY_LOCAL_MACHINE, r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search, 0, winreg.KEY_READ | arch_key, ) return Utils.joinpath(winreg.QueryValue(key, ""), "python.exe") except WindowsError: # lint:ok pass
def runScons(options, quiet): # For the scons file to find the static C++ files and include path. The # scons file is unable to use __file__ for the task. os.environ["NUITKA_SCONS"] = getSconsDataPath() if Utils.getOS() == "Windows": # On Windows this Scons variable must be set by us. os.environ["SCONS_LIB_DIR"] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.3.2" ) scons_command = getSconsBinaryCall() if quiet: scons_command.append("--quiet") scons_command += [ # The scons file "-f", Utils.joinpath(getSconsDataPath(), "SingleExe.scons"), # Parallel compilation. "--jobs", str(Options.getJobLimit()), # Do not warn about deprecations of Scons "--warn=no-deprecated", # Don't load "site_scons" at all. "--no-site-dir", ] if Options.isShowScons(): scons_command.append("--debug=explain") # Option values to provide to scons. for key, value in options.items(): scons_command += [key + "=" + value] if Options.isShowScons(): Tracing.printLine("Scons command:", " ".join(scons_command)) return 0 == subprocess.call(scons_command, shell = False)
def _consider(self, constraint_collection, module_filename, module_package): assert module_package is None or (type(module_package) is str and module_package != "") module_filename = Utils.normpath(module_filename) if Utils.isDir(module_filename): module_name = Utils.basename(module_filename) elif module_filename.endswith(".py"): module_name = Utils.basename(module_filename)[:-3] else: module_name = None if module_name is not None: decision = self._decide(module_filename, module_name, module_package) if decision: module_relpath = Utils.relpath(module_filename) return self._recurseTo( constraint_collection=constraint_collection, module_package=module_package, module_filename=module_filename, module_relpath=module_relpath) elif decision is None: if module_package is None: module_fullpath = module_name else: module_fullpath = module_package + "." + module_name if module_filename not in self._warned_about: self._warned_about.add(module_filename) warning( # long message, but shall be like it, pylint: disable=C0301 """\ Not recursing to '%(full_path)s' (%(filename)s), please specify \ --recurse-none (do not warn), \ --recurse-all (recurse to all), \ --recurse-not-to=%(full_path)s (ignore it), \ --recurse-to=%(full_path)s (recurse to it) to change.""" % { "full_path": module_fullpath, "filename": module_filename })
def checkPluginPath( plugin_filename, module_package ): plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: # File or package, handle that. if Utils.isFile( plugin_info[0] ) or Importing.isPackageDir( plugin_info[0] ): _checkPluginPath( plugin_filename, module_package ) elif Utils.isDir( plugin_info[0] ): for sub_path, sub_filename in Utils.listDir( plugin_info[0] ): assert sub_filename != "__init__.py" if Importing.isPackageDir( sub_path ) or sub_path.endswith( ".py" ): _checkPluginPath( sub_path, None ) else: warning( "Failed to include module from '%s'.", plugin_info[0] )
def getImportedModule(module_relpath): module_name = Utils.basename(module_relpath) if module_name.endswith(".py"): module_name = module_name[:-3] key = module_relpath, module_name return imported_modules[key]
def isImportedPath(module_relpath): module_name = Utils.basename(module_relpath) if module_name.endswith(".py"): module_name = module_name[:-3] key = module_relpath, module_name return key in imported_modules
def checkPluginPath(plugin_filename, module_package): plugin_info = considerFilename(module_package=module_package, module_filename=plugin_filename) if plugin_info is not None: # File or package, handle that. if Utils.isFile(plugin_info[0]) or Importing.isPackageDir( plugin_info[0]): _checkPluginPath(plugin_filename, module_package) elif Utils.isDir(plugin_info[0]): for sub_path, sub_filename in Utils.listDir(plugin_info[0]): assert sub_filename != "__init__.py" if Importing.isPackageDir(sub_path) or sub_path.endswith( ".py"): _checkPluginPath(sub_path, None) else: warning("Failed to include module from '%s'.", plugin_info[0])
def __init__(self, name, package_name, source_ref): NodeBase.__init__(self, source_ref=source_ref) PythonModuleMixin.__init__(self, name=name, package_name=package_name) assert Utils.basename(source_ref.getFilename()) != "<frozen>" # That is too likely a bug. assert name != "__main__"
def getImportedModule( module_relpath ): module_name = Utils.basename( module_relpath ) if module_name.endswith( ".py" ): module_name = module_name[:-3] key = module_relpath, module_name return imported_modules[ key ]
def isImportedPath( module_relpath ): module_name = Utils.basename( module_relpath ) if module_name.endswith( ".py" ): module_name = module_name[:-3] key = module_relpath, module_name return key in imported_modules
def _consider( self, constraint_collection, module_filename, module_package ): assert module_package is None or ( type( module_package ) is str and module_package != "" ) module_filename = Utils.normpath( module_filename ) if Utils.isDir( module_filename ): module_name = Utils.basename( module_filename ) elif module_filename.endswith( ".py" ): module_name = Utils.basename( module_filename )[:-3] else: module_name = None if module_name is not None: decision = self._decide( module_filename, module_name, module_package ) if decision: module_relpath = Utils.relpath( module_filename ) return self._recurseTo( constraint_collection = constraint_collection, module_package = module_package, module_filename = module_filename, module_relpath = module_relpath ) elif decision is None: if module_package is None: module_fullpath = module_name else: module_fullpath = module_package + "." + module_name if module_filename not in self._warned_about: self._warned_about.add( module_filename ) warning( # long message, but shall be like it, pylint: disable=C0301 """\ Not recursing to '%(full_path)s' (%(filename)s), please specify \ --recurse-none (do not warn), \ --recurse-all (recurse to all), \ --recurse-not-to=%(full_path)s (ignore it), \ --recurse-to=%(full_path)s (recurse to it) to change.""" % { "full_path" : module_fullpath, "filename" : module_filename } )
def detectBinaryDLLs(binary_filename, package_name): """ Detect the DLLs used by a binary. Using ldd (Linux), depends.exe (Windows), or otool (MacOS) the list of used DLLs is retrieved. """ if Utils.getOS() in ("Linux", "NetBSD"): # TODO: FreeBSD may work the same way, not tested. return _detectBinaryPathDLLsLinuxBSD(binary_filename=binary_filename) elif Utils.getOS() == "Windows": return _detectBinaryPathDLLsWindows(binary_filename=binary_filename, package_name=package_name) elif Utils.getOS() == "Darwin": return _detectBinaryPathDLLsLinuxBSD(binary_filename=binary_filename) else: # Support your platform above. assert False, Utils.getOS()
def _makeBinaryPathPathDLLSearchEnv(package_name): # Put the PYTHONPATH into the system "PATH", DLLs frequently live in # the package directories. env = os.environ.copy() path = env.get("PATH", "").split(';') # Put the "Python.exe" first. At least for WinPython, they put the DLLs # there. path = [sys.prefix] + sys.path + path if package_name is not None: for element in sys.path: candidate = Utils.joinpath(element, package_name) if Utils.isDir(candidate): path.append(candidate) env["PATH"] = ';'.join(path) return env
def runScons( options, quiet ): # For the scons file to find the static C++ files and include path. The scons file is # unable to use __file__ for the task. os.environ[ "NUITKA_SCONS" ] = getSconsDataPath() if os.name == "nt": # On Windows this Scons variable must be set by us. os.environ[ "SCONS_LIB_DIR" ] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.0.1" ) # Also, for MinGW we can avoid the user having to add the path if he used the # default path or installed it on the same drive by appending to the PATH variable # before executing scons. os.environ[ "PATH" ] += r";\MinGW\bin;C:\MinGW\bin" # Scons is Python2 only, so we need to make the system find a suitable Python binary. if Utils.python_version < 300: python_exe = sys.executable elif os.name == "nt": if os.path.exists( r"c:\Python27\python.exe" ): python_exe = r"c:\Python27\python.exe" elif os.path.exists( r"c:\Python26\python.exe" ): python_exe = r"c:\Python26\python.exe" else: sys.exit( """Error, need to find Python2 executable under C:\\Python26 or \ C:\\Python27 to execute scons which is not Python3 compatible.""" ) else: python_exe = "python" scons_command = """%(python)s %(binary)s %(quiet)s --warn=no-deprecated -f %(scons_file)s --jobs %(job_limit)d %(options)s""" % { "python" : python_exe, "binary" : getSconsBinaryPath(), "quiet" : "--quiet" if quiet else "", "scons_file" : Utils.joinpath( getSconsDataPath(), "SingleExe.scons" ), "job_limit" : Options.getJobLimit(), "options" : " ".join( "%s=%s" % ( key, value ) for key, value in options.items() ) } if Options.isShowScons(): Tracing.printLine( "Scons command:", scons_command ) return 0 == os.system( scons_command )
def getMainCode(codes, context): python_flags = Options.getPythonFlags() if context.isEmptyModule(): code_identifier = NullIdentifier() else: code_identifier = getCodeObjectHandle( context = context, filename = context.getFilename(), var_names = (), arg_count = 0, kw_only_count = 0, line_number = 0, code_name = "<module>", is_generator = False, is_optimized = False, has_starlist = False, has_stardict = False ) main_code = CodeTemplates.main_program % { "sys_executable" : getConstantCode( constant = "python.exe" if Utils.getOS() == "Windows" and \ Options.isStandaloneMode() else sys.executable, context = context ), "python_sysflag_debug" : sys.flags.debug, "python_sysflag_py3k_warning" : ( sys.flags.py3k_warning if hasattr( sys.flags, "py3k_warning" ) else 0 ), "python_sysflag_division_warning" : ( sys.flags.division_warning if hasattr( sys.flags, "division_warning" ) else 0 ), #"python_sysflag_division_new" : sys.flags.division_new, #not supported "python_sysflag_inspect" : sys.flags.inspect, "python_sysflag_interactive" : sys.flags.interactive, "python_sysflag_optimize" : sys.flags.optimize, "python_sysflag_dont_write_bytecode" : sys.flags.dont_write_bytecode, "python_sysflag_no_site" : sys.flags.no_site, "python_sysflag_no_user_site" : sys.flags.no_user_site, "python_sysflag_ignore_environment" : sys.flags.ignore_environment, "python_sysflag_tabcheck" : ( sys.flags.tabcheck if hasattr( sys.flags, "tabcheck" ) else 0 ), "python_sysflag_verbose" : 1 if "trace_imports" in python_flags else 0, "python_sysflag_unicode" : ( sys.flags.unicode if hasattr( sys.flags, "unicode" ) else 0 ), "python_sysflag_bytes_warning" : sys.flags.bytes_warning, "python_sysflag_hash_randomization" : ( sys.flags.hash_randomization if hasattr( sys.flags, "hash_randomization" ) and "no_randomization" not in python_flags else 0 ), "code_identifier" : code_identifier.getCodeTemporaryRef() } return codes + main_code
def isSameModulePath(path1, path2): if Utils.basename(path1) == "__init__.py": path1 = Utils.dirname(path1) if Utils.basename(path2) == "__init__.py": path2 = Utils.dirname(path2) return Utils.abspath(path1) == Utils.abspath(path2)
def __init__(self, name, package_name, source_ref): NodeBase.__init__( self, source_ref = source_ref ) PythonModuleMixin.__init__( self, name = name, package_name = package_name ) assert Utils.basename(source_ref.getFilename()) != "<frozen>"
def scanStandardLibraryPath(stdlib_dir): # There is a lot of black-listing here, done in branches, so there # is many of them, but that's acceptable, pylint: disable=R0912 for root, dirs, filenames in os.walk(stdlib_dir): import_path = root[len(stdlib_dir):].strip('/\\') import_path = import_path.replace("\\", ".").replace("/",".") if import_path == '': if 'site-packages' in dirs: dirs.remove('site-packages') if 'dist-packages' in dirs: dirs.remove('dist-packages') if 'test' in dirs: dirs.remove('test') if 'idlelib' in dirs: dirs.remove('idlelib') if 'turtledemo' in dirs: dirs.remove('turtledemo') if import_path in ('tkinter', 'importlib', 'ctypes'): if 'test' in dirs: dirs.remove('test') if import_path == "lib2to3": if 'tests' in dirs: dirs.remove('tests') if Utils.python_version >= 340 and Utils.getOS() == "Windows": if import_path == "multiprocessing": filenames.remove("popen_fork.py") filenames.remove("popen_forkserver.py") filenames.remove("popen_spawn_posix.py") for filename in filenames: if filename.endswith('.py') and filename not in ignore_modules: module_name = filename[:-3] if import_path == '': yield module_name else: yield import_path + '.' + module_name if Utils.python_version >= 300: if '__pycache__' in dirs: dirs.remove('__pycache__') for dirname in dirs: if import_path == '': yield dirname else: yield import_path + '.' + dirname
def _checkPluginPath( plugin_filename, module_package ): plugin_info = considerFilename( module_package = module_package, module_filename = plugin_filename ) if plugin_info is not None: module, added = recurseTo( module_filename = plugin_info[0], module_relpath = plugin_info[1], module_package = module_package ) if module: if not added: warning( "Recursed to %s '%s' at '%s' twice.", "package" if module.isPythonPackage() else "module", module.getName(), plugin_info[0] ) if module.isPythonPackage(): package_dir = Utils.dirname( module.getFilename() ) for sub_path, sub_filename in Utils.listDir( package_dir ): if sub_filename == "__init__.py": continue assert sub_path != plugin_filename, package_dir if Importing.isPackageDir( sub_path ) or sub_path.endswith( ".py" ): _checkPluginPath( sub_path, module.getFullName() ) else: warning( "Failed to include module from '%s'.", plugin_info[0] )
def __init__(self, name, package_name, source_ref): NodeBase.__init__( self, source_ref = source_ref ) PythonModuleMixin.__init__( self, name = name, package_name = package_name ) assert Utils.basename(source_ref.getFilename()) != "<frozen>" # That is too likely a bug. assert name != "__main__"
def _detectedSourceFile(filename, module_name, result, is_late): if module_name in module_names: return if module_name == "collections.abc": _detectedSourceFile( filename = filename, module_name = "_collections_abc", result = result, is_late = is_late ) source_code = open(filename,"rb").read() if Utils.python_version >= 300: source_code = source_code.decode("utf-8") filename = filename.decode("utf-8") if module_name == "site": source_code = """\ __file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % ( os.path.sep, os.path.basename(filename), source_code ) debug( "Freezing module '%s' (from '%s').", module_name, filename ) result.append( ( module_name, marshal.dumps( compile(source_code, filename, "exec") ), Utils.basename(filename) == "__init__.py", filename, is_late ) ) module_names.add(module_name)
def getPython2ExePath(): """ Find a way to call Python2. Scons needs it.""" if Utils.python_version < 300: return sys.executable elif Utils.getOS() == "Windows": python_exe = _getPython2ExePathWindows() if python_exe is not None: return python_exe else: sys.exit("""\ Error, need to find Python2 executable under C:\\Python26 or \ C:\\Python27 to execute scons which is not Python3 compatible.""") elif os.path.exists("/usr/bin/python2"): return "python2" else: return "python"
def attemptRecursion(self): # Make sure the package is recursed to. from nuitka.tree import Recursion from nuitka import Importing # Return the list of newly added modules. result = [] if self.package_name is not None and self.package is None: package_package, _package_module_name, package_filename = \ Importing.findModule( source_ref = self.getSourceReference(), module_name = self.package_name, parent_package = None, level = 1, warn = Utils.python_version < 330 ) # TODO: Temporary, if we can't find the package for Python3.3 that # is semi-OK, maybe. if Utils.python_version >= 330 and not package_filename: return [] imported_module, is_added = Recursion.recurseTo( module_package = package_package, module_filename = package_filename, module_relpath = Utils.relpath(package_filename), module_kind = "py", reason = "Containing package of recursed module.", ) self.package = imported_module if is_added: result.append(imported_module) if self.package: from nuitka.ModuleRegistry import addUsedModule addUsedModule(self.package) # print "Recursed to package", self.package_name result.extend(self.package.attemptRecursion()) return result
def copyUsedDLLs(dist_dir, binary_filename, standalone_entry_points): dll_map = [] for early_dll in detectUsedDLLs(standalone_entry_points): dll_name = Utils.basename(early_dll) target_path = Utils.joinpath( dist_dir, dll_name ) # Check that if a DLL has the name name, if it's identical, # happens at least for OSC and Fedora 20. if Utils.isFile(target_path): import filecmp if filecmp.cmp(early_dll, target_path): continue else: sys.exit("Error, conflicting DLLs for '%s'." % dll_name) shutil.copy( early_dll, target_path ) dll_map.append( (early_dll, dll_name) ) if Options.isShowInclusion(): info("Included used shared library '%s'.", early_dll) if Utils.getOS() == "Darwin": # For MacOS, the binary needs to be changed to reflect the DLL # location in the dist folder. fixupBinaryDLLPaths(binary_filename, dll_map) if Utils.getOS() == "Linux": # For Linux, the rpath of libraries may be an issue. for _original_path, early_dll in dll_map: removeSharedLibraryRPATH( Utils.joinpath(dist_dir, early_dll) ) for standalone_entry_point in standalone_entry_points[1:]: removeSharedLibraryRPATH( standalone_entry_point[0] )