def detectBinaryDLLs(binary_filename, package_name): """ Detect the DLLs used by a binary. Using ldd (Linux), depends.exe (Windows), or otool (MacOS) the list of used DLLs is retrieved. """ if Utils.getOS() in ("Linux", "NetBSD"): # TODO: FreeBSD may work the same way, not tested. return _detectBinaryPathDLLsLinuxBSD( binary_filename = binary_filename ) elif Utils.getOS() == "Windows": return _detectBinaryPathDLLsWindows( binary_filename = binary_filename, package_name = package_name ) elif Utils.getOS() == "Darwin": return _detectBinaryPathDLLsLinuxBSD( binary_filename = binary_filename ) else: # Support your platform above. assert False, Utils.getOS()
def copyUsedDLLs(dist_dir, binary_filename, standalone_entry_points): dll_map = [] for early_dll in detectUsedDLLs(standalone_entry_points): dll_name = Utils.basename(early_dll) target_path = Utils.joinpath( dist_dir, dll_name ) # Check that if a DLL has the name name, if it's identical, # happens at least for OSC and Fedora 20. if Utils.isFile(target_path): import filecmp if filecmp.cmp(early_dll, target_path): continue else: sys.exit("Error, conflicting DLLs for '%s'." % dll_name) shutil.copy( early_dll, target_path ) dll_map.append( (early_dll, dll_name) ) if Options.isShowInclusion(): info("Included used shared library '%s'.", early_dll) if Utils.getOS() == "Darwin": # For MacOS, the binary needs to be changed to reflect the DLL # location in the dist folder. fixupBinaryDLLPaths(binary_filename, dll_map) if Utils.getOS() == "Linux": # For Linux, the rpath of libraries may be an issue. for _original_path, early_dll in dll_map: removeSharedLibraryRPATH( Utils.joinpath(dist_dir, early_dll) ) for standalone_entry_point in standalone_entry_points[1:]: removeSharedLibraryRPATH( standalone_entry_point[0] )
def detectEarlyImports(): if Options.freezeAllStdlib(): stdlib_modules = set() # Scan the standard library paths (multiple in case of virtualenv. for stdlib_dir in getStandardLibraryPaths(): for module_name in scanStandardLibraryPath(stdlib_dir): stdlib_modules.add(module_name) import_code = 'imports = ' + repr(sorted(stdlib_modules)) + '\n'\ 'for imp in imports:\n' \ ' try:\n' \ ' __import__(imp)\n' \ ' except ImportError:\n' \ ' pass\n' else: # TODO: Should recursively include all of encodings module. import_code = "import encodings.utf_8;import encodings.ascii;import encodings.idna;" if Utils.getOS() == "Windows": import_code += "import encodings.mbcs;import encodings.cp437;" # String method hex depends on it. if Utils.python_version < 300: import_code += "import encodings.hex_codec;" import_code += "import locale;" result = _detectImports(import_code, False) debug("Finished detecting early imports.") return result
def getMainCode(codes, context): python_flags = Options.getPythonFlags() if context.isEmptyModule(): code_identifier = NullIdentifier() else: code_identifier = getCodeObjectHandle( context = context, filename = context.getFilename(), var_names = (), arg_count = 0, kw_only_count = 0, line_number = 0, code_name = "<module>", is_generator = False, is_optimized = False, has_starlist = False, has_stardict = False ) main_code = CodeTemplates.main_program % { "sys_executable" : getConstantCode( constant = "python.exe" if Utils.getOS() == "Windows" and \ Options.isStandaloneMode() else sys.executable, context = context ), "python_sysflag_debug" : sys.flags.debug, "python_sysflag_py3k_warning" : ( sys.flags.py3k_warning if hasattr( sys.flags, "py3k_warning" ) else 0 ), "python_sysflag_division_warning" : ( sys.flags.division_warning if hasattr( sys.flags, "division_warning" ) else 0 ), #"python_sysflag_division_new" : sys.flags.division_new, #not supported "python_sysflag_inspect" : sys.flags.inspect, "python_sysflag_interactive" : sys.flags.interactive, "python_sysflag_optimize" : sys.flags.optimize, "python_sysflag_dont_write_bytecode" : sys.flags.dont_write_bytecode, "python_sysflag_no_site" : sys.flags.no_site, "python_sysflag_no_user_site" : sys.flags.no_user_site, "python_sysflag_ignore_environment" : sys.flags.ignore_environment, "python_sysflag_tabcheck" : ( sys.flags.tabcheck if hasattr( sys.flags, "tabcheck" ) else 0 ), "python_sysflag_verbose" : 1 if "trace_imports" in python_flags else 0, "python_sysflag_unicode" : ( sys.flags.unicode if hasattr( sys.flags, "unicode" ) else 0 ), "python_sysflag_bytes_warning" : sys.flags.bytes_warning, "python_sysflag_hash_randomization" : ( sys.flags.hash_randomization if hasattr( sys.flags, "hash_randomization" ) and "no_randomization" not in python_flags else 0 ), "code_identifier" : code_identifier.getCodeTemporaryRef() } return codes + main_code
def scanStandardLibraryPath(stdlib_dir): # There is a lot of black-listing here, done in branches, so there # is many of them, but that's acceptable, pylint: disable=R0912 for root, dirs, filenames in os.walk(stdlib_dir): import_path = root[len(stdlib_dir):].strip('/\\') import_path = import_path.replace("\\", ".").replace("/",".") if import_path == '': if 'site-packages' in dirs: dirs.remove('site-packages') if 'dist-packages' in dirs: dirs.remove('dist-packages') if 'test' in dirs: dirs.remove('test') if 'idlelib' in dirs: dirs.remove('idlelib') if 'turtledemo' in dirs: dirs.remove('turtledemo') if import_path in ('tkinter', 'importlib', 'ctypes'): if 'test' in dirs: dirs.remove('test') if import_path == "lib2to3": if 'tests' in dirs: dirs.remove('tests') if Utils.python_version >= 340 and Utils.getOS() == "Windows": if import_path == "multiprocessing": filenames.remove("popen_fork.py") filenames.remove("popen_forkserver.py") filenames.remove("popen_spawn_posix.py") for filename in filenames: if filename.endswith('.py') and filename not in ignore_modules: module_name = filename[:-3] if import_path == '': yield module_name else: yield import_path + '.' + module_name if Utils.python_version >= 300: if '__pycache__' in dirs: dirs.remove('__pycache__') for dirname in dirs: if import_path == '': yield dirname else: yield import_path + '.' + dirname
def runScons(options, quiet): # For the scons file to find the static C++ files and include path. The # scons file is unable to use __file__ for the task. os.environ["NUITKA_SCONS"] = getSconsDataPath() if Utils.getOS() == "Windows": # On Windows this Scons variable must be set by us. os.environ["SCONS_LIB_DIR"] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.3.0" ) # Also, for MinGW we can avoid the user having to add the path if he # used the default path or installed it on the same drive by appending # to the PATH variable before executing scons. os.environ["PATH"] += r";\MinGW\bin;C:\MinGW\bin" scons_command = getSconsBinaryCall() if quiet: scons_command.append("--quiet") scons_command += [ # The scons file "-f", Utils.joinpath(getSconsDataPath(), "SingleExe.scons"), # Parallel compilation. "--jobs", str(Options.getJobLimit()), # Do not warn about deprecations of Scons "--warn=no-deprecated", # Don't load "site_scons" at all. "--no-site-dir", ] if Options.isShowScons(): scons_command.append("--debug=explain") # Option values to provide to scons. for key, value in options.items(): scons_command += [key + "=" + value] if Options.isShowScons(): Tracing.printLine("Scons command:", " ".join(scons_command)) return 0 == subprocess.call(scons_command)
def getPython2ExePath(): """ Find a way to call Python2. Scons needs it.""" if Utils.python_version < 300: return sys.executable elif Utils.getOS() == "Windows": python_exe = _getPython2ExePathWindows() if python_exe is not None: return python_exe else: sys.exit("""\ Error, need to find Python2 executable under C:\\Python26 or \ C:\\Python27 to execute scons which is not Python3 compatible.""") elif os.path.exists("/usr/bin/python2"): return "python2" else: return "python"
def runScons(options, quiet): # For the scons file to find the static C++ files and include path. The # scons file is unable to use __file__ for the task. os.environ["NUITKA_SCONS"] = getSconsDataPath() if Utils.getOS() == "Windows": # On Windows this Scons variable must be set by us. os.environ["SCONS_LIB_DIR"] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.3.2" ) scons_command = getSconsBinaryCall() if quiet: scons_command.append("--quiet") scons_command += [ # The scons file "-f", Utils.joinpath(getSconsDataPath(), "SingleExe.scons"), # Parallel compilation. "--jobs", str(Options.getJobLimit()), # Do not warn about deprecations of Scons "--warn=no-deprecated", # Don't load "site_scons" at all. "--no-site-dir", ] if Options.isShowScons(): scons_command.append("--debug=explain") # Option values to provide to scons. for key, value in options.items(): scons_command += [key + "=" + value] if Options.isShowScons(): Tracing.printLine("Scons command:", " ".join(scons_command)) return 0 == subprocess.call(scons_command, shell = False)
def detectBinaryDLLs(binary_filename, package_name): result = set() if Utils.getOS() in ("Linux", "NetBSD"): # Ask "ldd" about the libraries being used by the created binary, these # are the ones that interest us. process = subprocess.Popen( args = [ "ldd", binary_filename ], stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout, _stderr = process.communicate() for line in stdout.split(b"\n"): if not line: continue if b"=>" not in line: continue part = line.split(b" => ", 2)[1] if b"(" in part: filename = part[:part.rfind(b"(")-1] else: filename = part if not filename: continue if Utils.python_version >= 300: filename = filename.decode("utf-8") result.add(filename) elif Utils.getOS() == "Windows": depends_exe = getDependsExePath() env = os.environ.copy() path = env.get("PATH","").split(";") path += sys.path if package_name is not None: for element in sys.path: candidate = Utils.joinpath(element, package_name) if Utils.isDir(candidate): path.append(candidate) env["PATH"] = ";".join(path) subprocess.call( ( depends_exe, "-c", "-ot%s" % binary_filename + ".depends", "-f1", "-pa1", "-ps1", binary_filename ), env = env, ) inside = False for line in open(binary_filename + ".depends"): if "| Module Dependency Tree |" in line: inside = True continue if not inside: continue if "| Module List |" in line: break if "]" not in line: continue # Skip missing DLLs, apparently not needed anyway. if "?" in line[:line.find("]")]: continue dll_filename = line[line.find("]")+2:-1] assert Utils.isFile(dll_filename), dll_filename # The executable itself is of course excempted. if Utils.normcase(dll_filename) == \ Utils.normcase(Utils.abspath(binary_filename)): continue dll_name = Utils.basename(dll_filename).upper() # Win API can be assumed. if dll_name.startswith("API-MS-WIN-") or dll_name.startswith("EXT-MS-WIN-"): continue if dll_name in ("SHELL32.DLL", "USER32.DLL", "KERNEL32.DLL", "NTDLL.DLL", "NETUTILS.DLL", "LOGONCLI.DLL", "GDI32.DLL", "RPCRT4.DLL", "ADVAPI32.DLL", "SSPICLI.DLL", "SECUR32.DLL", "KERNELBASE.DLL", "WINBRAND.DLL", "DSROLE.DLL", "DNSAPI.DLL", "SAMCLI.DLL", "WKSCLI.DLL", "SAMLIB.DLL", "WLDAP32.DLL", "NTDSAPI.DLL", "CRYPTBASE.DLL", "W32TOPL", "WS2_32.DLL", "SPPC.DLL", "MSSIGN32.DLL", "CERTCLI.DLL", "WEBSERVICES.DLL", "AUTHZ.DLL", "CERTENROLL.DLL", "VAULTCLI.DLL", "REGAPI.DLL", "BROWCLI.DLL", "WINNSI.DLL", "DHCPCSVC6.DLL", "PCWUM.DLL", "CLBCATQ.DLL", "IMAGEHLP.DLL", "MSASN1.DLL", "DBGHELP.DLL", "DEVOBJ.DLL", "DRVSTORE.DLL", "CABINET.DLL", "SCECLI.DLL", "SPINF.DLL", "SPFILEQ.DLL", "GPAPI.DLL", "NETJOIN.DLL", "W32TOPL.DLL", "NETBIOS.DLL", "DXGI.DLL", "DWRITE.DLL", "D3D11.DLL", "WLANAPI.DLL", "WLANUTIL.DLL", "ONEX.DLL", "EAPPPRXY.DLL", "MFPLAT.DLL", "AVRT.DLL", "ELSCORE.DLL", "INETCOMM.DLL", "MSOERT2.DLL", "IEUI.DLL", "MSCTF.DLL", "MSFEEDS.DLL", "UIAUTOMATIONCORE.DLL", "PSAPI.DLL", "EFSADU.DLL", "MFC42U.DLL", "ODBC32.DLL", "OLEDLG.DLL", "NETAPI32.DLL", "LINKINFO.DLL", "DUI70.DLL", "ADVPACK.DLL", "NTSHRUI.DLL", "WINSPOOL.DRV", "EFSUTIL.DLL", "WINSCARD.DLL", "SHDOCVW.DLL", "IEFRAME.DLL", "D2D1.DLL", "GDIPLUS.DLL", "OCCACHE.DLL", "IEADVPACK.DLL", "MLANG.DLL", "MSI.DLL", "MSHTML.DLL", "COMDLG32.DLL", "PRINTUI.DLL", "PUIAPI.DLL", "ACLUI.DLL", "WTSAPI32.DLL", "FMS.DLL", "DFSCLI.DLL", "HLINK.DLL", "MSRATING.DLL", "PRNTVPT.DLL", "IMGUTIL.DLL", "MSLS31.DLL", "VERSION.DLL", "NORMALIZ.DLL", "IERTUTIL.DLL", "WININET.DLL", "WINTRUST.DLL", "XMLLITE.DLL", "APPHELP.DLL", "PROPSYS.DLL", "RSTRTMGR.DLL", "NCRYPT.DLL", "BCRYPT.DLL", "MMDEVAPI.DLL", "MSILTCFG.DLL", "DEVMGR.DLL", "DEVRTL.DLL", "NEWDEV.DLL", "VPNIKEAPI.DLL", "WINHTTP.DLL", "WEBIO.DLL", "NSI.DLL", "DHCPCSVC.DLL", "CRYPTUI.DLL", "ESENT.DLL", "DAVHLPR.DLL", "CSCAPI.DLL", "ATL.DLL", "OLEAUT32.DLL", "SRVCLI.DLL", "RASDLG.DLL", "MPRAPI.DLL", "RTUTILS.DLL", "RASMAN.DLL", "MPRMSG.DLL", "SLC.DLL", "CRYPTSP.DLL", "RASAPI32.DLL", "TAPI32.DLL", "EAPPCFG.DLL", "NDFAPI.DLL", "WDI.DLL", "COMCTL32.DLL", "UXTHEME.DLL", "IMM32.DLL", "OLEACC.DLL", "WINMM.DLL", "WINDOWSCODECS.DLL", "DWMAPI.DLL", "DUSER.DLL", "PROFAPI.DLL", "URLMON.DLL", "SHLWAPI.DLL", "LPK.DLL", "USP10.DLL", "CFGMGR32.DLL", "MSIMG32.DLL", "POWRPROF.DLL", "SETUPAPI.DLL", "WINSTA.DLL", "CRYPT32.DLL", "IPHLPAPI.DLL", "MPR.DLL", "CREDUI.DLL", "NETPLWIZ.DLL", "OLE32.DLL", "ACTIVEDS.DLL", "ADSLDPC.DLL", "USERENV.DLL", "APPREPAPI.DLL", "BCP47LANGS.DLL", "BCRYPTPRIMITIVES.DLL", "CERTCA.DLL", "CHARTV.DLL", "COMBASE.DLL", "DCOMP.DLL", "DPAPI.DLL", "DSPARSE.DLL", "FECLIENT.DLL", "FIREWALLAPI.DLL", "FLTLIB.DLL", "MRMCORER.DLL", "MSVCRT.DLL", "NINPUT.DLL", "NTASN1.DLL", "PCACLI.DLL", "RTWORKQ.DLL", "SECHOST.DLL", "SETTINGSYNCPOLICY.DLL", "SHCORE.DLL", "TBS.DLL", "TWINAPI.DLL", "TWINAPI.APPCORE.DLL", "VIRTDISK.DLL", "WEBSOCKET.DLL", "WEVTAPI.DLL", "WINMMBASE.DLL", "WMICLNT.DLL"): continue result.add(dll_filename) os.unlink(binary_filename + ".depends") else: # Support your platform above. assert False, Utils.getOS() return result
def detectEarlyImports(): if Options.freezeAllStdlib(): stdlib_modules = [] stdlib_dir = os.path.dirname(os.__file__) ignore_modules = [ "__main__.py", "__init__.py", "antigravity.py", ] if os.name != "nt": ignore_modules.append("wintypes.py") ignore_modules.append("cp65001.py") for root, dirs, filenames in os.walk(stdlib_dir): import_path = root[len(stdlib_dir):].strip('/\\') if import_path == '': if 'site-packages' in dirs: dirs.remove('site-packages') if 'dist-packages' in dirs: dirs.remove('dist-packages') if 'test' in dirs: dirs.remove('test') if 'idlelib' in dirs: dirs.remove('idlelib') if 'turtledemo' in dirs: dirs.remove('turtledemo') if import_path in ('tkinter', 'importlib'): if 'test' in dirs: dirs.remove('test') for filename in filenames: if filename.endswith('.py') and filename not in ignore_modules: module_name = filename[:-3] if import_path == '': stdlib_modules.append(module_name) else: stdlib_modules.append(import_path + '.' + module_name) if Utils.python_version >= 300: if '__pycache__' in dirs: dirs.remove('__pycache__') for dir in dirs: if import_path == '': stdlib_modules.append(dir) else: stdlib_modules.append(import_path + '.' + dir) import_code = 'imports = ' + repr(sorted(stdlib_modules)) + '\n'\ 'for imp in imports:\n' \ ' try:\n' \ ' __import__(imp)\n' \ ' except ImportError:\n' \ ' pass\n' else: # TODO: Should recursively include all of encodings module. import_code = "import encodings.utf_8;import encodings.ascii;" if Utils.getOS() == "Windows": import_code += "import encodings.mbcs;import encodings.cp437;" # String method hex depends on it. if Utils.python_version < 300: import_code += "import encodings.hex_codec;" import_code += "import locale;" result = _detectImports(import_code, False) debug("Finished detecting early imports.") return result
def copyUsedDLLs(dist_dir, binary_filename, standalone_entry_points): # This is terribly complex, because we check the list of used DLLs # trying to avoid duplicates, and detecting errors with them not # being binary identical, so we can report them. And then of course # we also need to handle OS specifics, pylint: disable=R0912,R0914 dll_map = [] used_dlls = detectUsedDLLs(standalone_entry_points) for dll_filename1, sources1 in tuple(iterItems(used_dlls)): for dll_filename2, sources2 in tuple(iterItems(used_dlls)): if dll_filename1 == dll_filename2: continue # Colliding basenames are an issue to us. if Utils.basename(dll_filename1) != Utils.basename(dll_filename2): continue # May already have been removed earlier if dll_filename1 not in used_dlls: continue if dll_filename2 not in used_dlls: continue dll_name = Utils.basename(dll_filename1) if Options.isShowInclusion(): info( """Colliding DLL names for %s, checking identity of \ '%s' <-> '%s'.""" % ( dll_name, dll_filename1, dll_filename2, ) ) # Check that if a DLL has the same name, if it's identical, # happens at least for OSC and Fedora 20. import filecmp if filecmp.cmp(dll_filename1, dll_filename2): del used_dlls[dll_filename2] continue sys.exit( """Error, conflicting DLLs for '%s' \ (%s used by %s different from %s used by %s).""" % ( dll_name, dll_filename1, ", ".join(sources1), dll_filename2, ", ".join(sources2) ) ) for dll_filename, sources in iterItems(used_dlls): dll_name = Utils.basename(dll_filename) target_path = Utils.joinpath( dist_dir, dll_name ) shutil.copy( dll_filename, target_path ) dll_map.append( (dll_filename, dll_name) ) if Options.isShowInclusion(): info( "Included used shared library '%s' (used by %s)." % ( dll_filename, ", ".join(sources) ) ) if Utils.getOS() == "Darwin": # For MacOS, the binary needs to be changed to reflect the DLL # location in the dist folder. fixupBinaryDLLPaths(binary_filename, dll_map) if Utils.getOS() == "Linux": # For Linux, the rpath of libraries may be an issue. for _original_path, dll_filename in dll_map: removeSharedLibraryRPATH( Utils.joinpath(dist_dir, dll_filename) ) for standalone_entry_point in standalone_entry_points[1:]: removeSharedLibraryRPATH( standalone_entry_point[0] )
def copyUsedDLLs(dist_dir, standalone_entry_points): # This is terribly complex, because we check the list of used DLLs # trying to avoid duplicates, and detecting errors with them not # being binary identical, so we can report them. And then of course # we also need to handle OS specifics, pylint: disable=R0912 dll_map = [] used_dlls = detectUsedDLLs(standalone_entry_points) for dll_filename1, sources1 in tuple(iterItems(used_dlls)): for dll_filename2, sources2 in tuple(iterItems(used_dlls)): if dll_filename1 == dll_filename2: continue # Colliding basenames are an issue to us. if Utils.basename(dll_filename1) != Utils.basename(dll_filename2): continue # May already have been removed earlier if dll_filename1 not in used_dlls: continue if dll_filename2 not in used_dlls: continue dll_name = Utils.basename(dll_filename1) if Options.isShowInclusion(): info("""Colliding DLL names for %s, checking identity of \ '%s' <-> '%s'.""" % ( dll_name, dll_filename1, dll_filename2, )) # Check that if a DLL has the same name, if it's identical, # happens at least for OSC and Fedora 20. import filecmp if filecmp.cmp(dll_filename1, dll_filename2): del used_dlls[dll_filename2] continue sys.exit("""Error, conflicting DLLs for '%s' \ (%s used by %s different from %s used by %s).""" % (dll_name, dll_filename1, ", ".join(sources1), dll_filename2, ", ".join(sources2))) for dll_filename, sources in iterItems(used_dlls): dll_name = Utils.basename(dll_filename) target_path = Utils.joinpath(dist_dir, dll_name) shutil.copy(dll_filename, target_path) dll_map.append((dll_filename, dll_name)) if Options.isShowInclusion(): info("Included used shared library '%s' (used by %s)." % (dll_filename, ", ".join(sources))) if Utils.getOS() == "Darwin": # For MacOS, the binary and the DLLs needs to be changed to reflect # the relative DLL location in the ".dist" folder. for standalone_entry_point in standalone_entry_points: fixupBinaryDLLPaths( binary_filename=standalone_entry_point[0], is_exe=standalone_entry_point is standalone_entry_points[0], dll_map=dll_map) for _original_path, dll_filename in dll_map: fixupBinaryDLLPaths(binary_filename=Utils.joinpath( dist_dir, dll_filename), is_exe=False, dll_map=dll_map) if Utils.getOS() == "Linux": # For Linux, the "rpath" of libraries may be an issue and must be # removed. for _original_path, dll_filename in dll_map: removeSharedLibraryRPATH(Utils.joinpath(dist_dir, dll_filename)) for standalone_entry_point in standalone_entry_points[1:]: removeSharedLibraryRPATH(standalone_entry_point[0])
result = _detectImports(command, True) debug("Finished detecting late imports.") return result else: return "" # Some modules we want to blacklist. ignore_modules = [ "__main__.py", "__init__.py", "antigravity.py", ] if Utils.getOS() != "Windows": ignore_modules.append("wintypes.py") ignore_modules.append("cp65001.py") def scanStandardLibraryPath(stdlib_dir): # There is a lot of black-listing here, done in branches, so there # is many of them, but that's acceptable, pylint: disable=R0912 for root, dirs, filenames in os.walk(stdlib_dir): import_path = root[len(stdlib_dir):].strip("/\\") import_path = import_path.replace('\\', '.').replace('/', '.') if import_path == "": if "site-packages" in dirs: dirs.remove("site-packages")