def detectDLLsWithDependencyWalker(binary_filename, scan_dirs): dwp_filename = binary_filename + ".dwp" output_filename = binary_filename + ".depends" # User query should only happen once if at all. with withFileLock( "Finding out dependency walker path and creating DWP file for %s" % binary_filename ): depends_exe = getDependsExePath() # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. putTextFileContents( dwp_filename, contents="""\ %(scan_dirs)s SxS """ % { "scan_dirs": "\n".join( "UserDir %s" % getExternalUsePath(dirname) for dirname in scan_dirs ) }, ) # Starting the process while locked, so file handles are not duplicated. # TODO: At least exit code should be checked, output goes to a filename, # but errors might be interesting potentially. _stdout, _stderr, _exit_code = executeProcess( command=( depends_exe, "-c", "-ot%s" % output_filename, "-d:%s" % dwp_filename, "-f1", "-pa1", "-ps1", binary_filename, ), external_cwd=True, ) if not os.path.exists(output_filename): inclusion_logger.sysexit( "Error, 'depends.exe' failed to produce expected output." ) # Opening the result under lock, so it is not getting locked by new processes. # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. result = parseDependsExeOutput(output_filename) deleteFile(output_filename, must_exist=True) deleteFile(dwp_filename, must_exist=True) return result
def detectDLLsWithDependencyWalker(binary_filename, scan_dirs): dwp_filename = binary_filename + ".dwp" output_filename = binary_filename + ".depends" # User query should only happen once if at all. with withFileLock( "Finding out dependency walker path and creating DWP file for %s" % binary_filename): depends_exe = getDependsExePath() # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. with open(dwp_filename, "w") as dwp_file: dwp_file.write( """\ %(scan_dirs)s SxS """ % { "scan_dirs": "\n".join("UserDir %s" % getExternalUsePath(dirname) for dirname in scan_dirs) }) # Starting the process while locked, so file handles are not duplicated. depends_exe_process = subprocess.Popen( ( depends_exe, "-c", "-ot%s" % output_filename, "-d:%s" % dwp_filename, "-f1", "-pa1", "-ps1", binary_filename, ), stdin=getNullInput(), cwd=getExternalUsePath(os.getcwd()), ) # TODO: Exit code should be checked. depends_exe_process.wait() if not os.path.exists(output_filename): inclusion_logger.sysexit( "Error, depends.exe failed to produce expected output.") # Opening the result under lock, so it is not getting locked by new processes. # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. result = _parseDependsExeOutput(output_filename) deleteFile(output_filename, must_exist=True) deleteFile(dwp_filename, must_exist=True) return result
def cleanSourceDirectory(source_dir): extensions = (".bin", ".c", ".cpp", ".exp", ".h", ".lib", ".manifest", ".o", ".obj", ".os", ".rc", ".res", ".S") if os.path.isdir(source_dir): for path, _filename in listDir(source_dir): if hasFilenameExtension(path, extensions): deleteFile(path, must_exist=True) else: makePath(source_dir)
def _deleteMsvcPGOFiles(pgo_mode): assert _wasMsvcMode() msvc_pgc_filename = OutputDirectories.getResultBasepath(onefile=False) + "!1.pgc" deleteFile(msvc_pgc_filename, must_exist=False) if pgo_mode == "use": msvc_pgd_filename = OutputDirectories.getResultBasepath(onefile=False) + ".pgd" deleteFile(msvc_pgd_filename, must_exist=False) return msvc_pgc_filename
def createNodeTree(filename): """ Create a node tree. Turn that source code into a node tree structure. If recursion into imported modules is available, more trees will be available during optimization, or immediately through recursed directory paths. """ # First, build the raw node tree from the source code. main_module = Building.buildModuleTree( filename=filename, package=None, is_top=True, is_main=not Options.shallMakeModule()) ModuleRegistry.addRootModule(main_module) # First remove old object files and old generated files, old binary or # module, and standalone mode program directory if any, they can only do # harm. source_dir = getSourceDirectoryPath(main_module) if not Options.shallOnlyExecCppCall(): cleanSourceDirectory(source_dir) # Prepare the ".dist" directory, throwing away what was there before. if Options.isStandaloneMode(): standalone_dir = getStandaloneDirectoryPath(main_module) removeDirectory(path=standalone_dir, ignore_errors=True) makePath(standalone_dir) deleteFile(path=getResultFullpath(main_module), must_exist=False) # Second, do it for the directories given. for plugin_filename in Options.getShallFollowExtra(): Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None) for pattern in Options.getShallFollowExtraFilePatterns(): Recursion.checkPluginFilenamePattern(pattern=pattern) # Then optimize the tree and potentially recursed modules. Optimization.optimize() if Options.isExperimental(): for module in ModuleRegistry.getRootModules(): if module.isMainModule(): return module assert False else: # Main module might change behind our back, look it up again. return main_module
def cleanSourceDirectory(source_dir): extensions = ( ".bin", ".c", ".cpp", ".exp", ".h", ".lib", ".manifest", ".o", ".obj", ".os", ".rc", ".res", ".S", ".txt", ) if os.path.isdir(source_dir): for path, _filename in listDir(source_dir): if hasFilenameExtension(path, extensions): deleteFile(path, must_exist=True) else: makePath(source_dir)
def _createNodeTree(filename): """Create a node tree. Turn that source code into a node tree structure. If recursion into imported modules is available, more trees will be available during optimization, or immediately through recursed directory paths. """ # Many cases to deal with, pylint: disable=too-many-branches # First, build the raw node tree from the source code. main_module = Building.buildMainModuleTree( filename=filename, is_main=not Options.shallMakeModule(), ) # First remove old object files and old generated files, old binary or # module, and standalone mode program directory if any, they can only do # harm. source_dir = OutputDirectories.getSourceDirectoryPath() if not Options.shallOnlyExecCCompilerCall(): SconsInterface.cleanSconsDirectory(source_dir) # Prepare the ".dist" directory, throwing away what was there before. if Options.isStandaloneMode(): standalone_dir = OutputDirectories.getStandaloneDirectoryPath(bundle=False) removeDirectory(path=standalone_dir, ignore_errors=True) if Options.shallCreateAppBundle(): removeDirectory( path=changeFilenameExtension(standalone_dir, ".app"), ignore_errors=True ) # Delete result file, to avoid confusion with previous build and to # avoid locking issues after the build. deleteFile( path=OutputDirectories.getResultFullpath(onefile=False), must_exist=False ) if Options.isOnefileMode(): deleteFile( path=OutputDirectories.getResultFullpath(onefile=True), must_exist=False ) # Second, do it for the directories given. for plugin_filename in Options.getShallFollowExtra(): Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None) for pattern in Options.getShallFollowExtraFilePatterns(): Recursion.checkPluginFilenamePattern(pattern=pattern) for package_name in Options.getMustIncludePackages(): package_name, package_directory, kind = Importing.locateModule( module_name=ModuleName(package_name), parent_package=None, level=0, ) if kind != "absolute": inclusion_logger.sysexit( "Error, failed to locate package %r you asked to include." % package_name.asString() ) Recursion.checkPluginPath( plugin_filename=package_directory, module_package=package_name.getPackageName(), ) for module_name in Options.getMustIncludeModules(): module_name, module_filename, kind = Importing.locateModule( module_name=ModuleName(module_name), parent_package=None, level=0, ) if kind != "absolute": inclusion_logger.sysexit( "Error, failed to locate module '%s' you asked to include." % module_name.asString() ) Recursion.checkPluginSinglePath( plugin_filename=module_filename, module_package=module_name.getPackageName() ) # Allow plugins to add more modules based on the initial set being complete. Plugins.onModuleInitialSet() # Then optimize the tree and potentially recursed modules. optimizeModules(main_module.getOutputFilename()) # Allow plugins to comment on final module set. Plugins.onModuleCompleteSet() if Options.isExperimental("check_xml_persistence"): for module in ModuleRegistry.getRootModules(): if module.isMainModule(): return module assert False else: # Main module might change behind our back, look it up again. return main_module
def _detectBinaryPathDLLsWindows(is_main_executable, source_dir, original_dir, binary_filename, package_name): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-locals result = set() cache_filename = _getCacheFilename(is_main_executable, source_dir, original_dir, binary_filename) if os.path.exists(cache_filename ) and not Options.shallNotUseDependsExeCachedResults(): for line in open(cache_filename): line = line.strip() result.add(line) return result # User query should only happen once if at all. with _withLock(): depends_exe = getDependsExePath() scan_dirs = [sys.prefix] if package_name is not None: from nuitka.importing.Importing import findModule package_dir = findModule(None, package_name, None, 0, False)[1] if os.path.isdir(package_dir): scan_dirs.append(package_dir) scan_dirs.extend(getSubDirectories(package_dir)) if original_dir is not None: scan_dirs.append(original_dir) scan_dirs.extend(getSubDirectories(original_dir)) with _withLock(): # The search order by default prefers the system directory, where a # wrong "PythonXX.dll" might be living. with open(binary_filename + ".dwp", 'w') as dwp_file: dwp_file.write( """\ KnownDLLs %(original_dirs)s SysPath 32BitSysDir 16BitSysDir OSDir AppPath SxS """ % { "original_dirs": '\n'.join("UserDir %s" % dirname for dirname in scan_dirs if not os.path.basename(dirname) == "__pycache__" if any(entry[1].lower().endswith(".dll") for entry in listDir(dirname))), }) # Starting the process while locked, so file handles are not duplicated. depends_exe_process = subprocess.Popen( (depends_exe, "-c", "-ot%s" % binary_filename + ".depends", "-d:%s" % binary_filename + ".dwp", "-f1", "-pa1", "-ps1", binary_filename)) # TODO: Exit code should be checked. depends_exe_process.wait() # Opening the result under lock, so it is not getting locked by new processes. with _withLock(): _parseDependsExeOutput(binary_filename + ".depends", result) deleteFile(binary_filename + ".depends", must_exist=True) deleteFile(binary_filename + ".dwp", must_exist=True) if not Options.shallNotStoreDependsExeCachedResults(): with open(cache_filename, 'w') as cache_file: for dll_filename in result: print(dll_filename, file=cache_file) return result
def getDependsExePath(): """ Return the path of depends.exe (for Windows). Will prompt the user to download if not already cached in AppData directory for Nuitka. """ if Utils.getArchitecture() == "x86": depends_url = "http://dependencywalker.com/depends22_x86.zip" else: depends_url = "http://dependencywalker.com/depends22_x64.zip" nuitka_app_dir = getAppDir() nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture()) nuitka_depends_zip = os.path.join(nuitka_depends_dir, os.path.basename(depends_url)) depends_exe = os.path.join(nuitka_depends_dir, "depends.exe") makePath(nuitka_depends_dir) if not os.path.isfile(nuitka_depends_zip) and not os.path.isfile( depends_exe): if assumeYesForDownloads(): reply = "y" else: Tracing.printLine("""\ Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool to analyze the dependencies of Python extension modules. Is it OK to download and put it in "%s". No installer needed, cached, one time question. Proceed and download? [Yes]/No """ % (nuitka_app_dir)) Tracing.flushStdout() reply = raw_input() if reply.lower() in ("no", "n"): sys.exit( "Nuitka does not work in --standalone on Windows without.") info("Downloading '%s'" % depends_url) try: urlretrieve(depends_url, nuitka_depends_zip) except Exception: # Any kind of error, pylint: disable=broad-except sys.exit("""Failed to download '%s'.\ Contents should manually be extracted to '%s'.""" % (depends_url, nuitka_depends_dir)) if not os.path.isfile(depends_exe): info("Extracting to '%s'" % depends_exe) import zipfile try: depends_zip = zipfile.ZipFile(nuitka_depends_zip) depends_zip.extractall(nuitka_depends_dir) except Exception: # Catching anything zip throws, pylint: disable=broad-except info("Problem with the downloaded zip file, deleting it.") deleteFile(depends_exe, must_exist=False) deleteFile(nuitka_depends_zip, must_exist=True) sys.exit("Error, need '%s' as extracted from '%s'." % (depends_exe, depends_url)) assert os.path.isfile(depends_exe) return depends_exe
def detectBinaryPathDLLsWindowsDependencyWalker( is_main_executable, source_dir, original_dir, binary_filename, package_name, use_cache, update_cache, ): # This is complex, as it also includes the caching mechanism # pylint: disable=too-many-locals result = set() if use_cache or update_cache: cache_filename = _getCacheFilename( dependency_tool="depends.exe", is_main_executable=is_main_executable, source_dir=source_dir, original_dir=original_dir, binary_filename=binary_filename, ) if use_cache: with withFileLock(): if not os.path.exists(cache_filename): use_cache = False if use_cache: for line in getFileContentByLine(cache_filename): line = line.strip() result.add(line) return result if Options.isShowProgress(): info("Analysing dependencies of '%s'." % binary_filename) scan_dirs = getScanDirectories(package_name, original_dir) dwp_filename = binary_filename + ".dwp" output_filename = binary_filename + ".depends" # User query should only happen once if at all. with _withLock( "Finding out dependency walker path and creating DWP file for %s" % binary_filename ): depends_exe = getDependsExePath() # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. with open(dwp_filename, "w") as dwp_file: dwp_file.write( """\ %(scan_dirs)s SxS """ % { "scan_dirs": "\n".join( "UserDir %s" % dirname for dirname in scan_dirs ) } ) # Starting the process while locked, so file handles are not duplicated. depends_exe_process = subprocess.Popen( ( depends_exe, "-c", "-ot%s" % output_filename, "-d:%s" % dwp_filename, "-f1", "-pa1", "-ps1", binary_filename, ), cwd=getExternalUsePath(os.getcwd()), ) # TODO: Exit code should be checked. depends_exe_process.wait() if not os.path.exists(output_filename): sys.exit("Error, depends.exe failed to product output.") # Opening the result under lock, so it is not getting locked by new processes. # Note: Do this under lock to avoid forked processes to hold # a copy of the file handle on Windows. _parseDependsExeOutput(output_filename, result) deleteFile(output_filename, must_exist=True) deleteFile(dwp_filename, must_exist=True) if update_cache: with open(cache_filename, "w") as cache_file: for dll_filename in result: print(dll_filename, file=cache_file) return result
def getDependsExePath(): """ Return the path of depends.exe (for Windows). Will prompt the user to download if not already cached in AppData directory for Nuitka. """ if Utils.getArchitecture() == "x86": depends_url = "http://dependencywalker.com/depends22_x86.zip" else: depends_url = "http://dependencywalker.com/depends22_x64.zip" nuitka_app_dir = getAppDir() nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture()) nuitka_depends_zip = os.path.join(nuitka_depends_dir, os.path.basename(depends_url)) depends_exe = os.path.join(nuitka_depends_dir, "depends.exe") makePath(nuitka_depends_dir) if not os.path.isfile(nuitka_depends_zip) and not os.path.isfile(depends_exe): if assumeYesForDownloads(): reply = "y" else: Tracing.printLine( """\ Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool to analyze the dependencies of Python extension modules. Is it OK to download and put it in "%s". No installer needed, cached, one time question. Proceed and download? [Yes]/No """ % (nuitka_app_dir) ) Tracing.flushStdout() reply = raw_input() if reply.lower() in ("no", "n"): sys.exit("Nuitka does not work in --standalone on Windows without.") info("Downloading '%s'" % depends_url) try: urlretrieve(depends_url, nuitka_depends_zip) except Exception: # Any kind of error, pylint: disable=broad-except sys.exit( """Failed to download '%s'.\ Contents should manually be extracted to '%s'.""" % (depends_url, nuitka_depends_dir) ) if not os.path.isfile(depends_exe): info("Extracting to '%s'" % depends_exe) import zipfile try: depends_zip = zipfile.ZipFile(nuitka_depends_zip) depends_zip.extractall(nuitka_depends_dir) except Exception: # Catching anything zip throws, pylint:disable=W0703 info("Problem with the downloaded zip file, deleting it.") deleteFile(depends_exe, must_exist=False) deleteFile(nuitka_depends_zip, must_exist=True) sys.exit( "Error, need '%s' as extracted from '%s'." % (depends_exe, depends_url) ) assert os.path.isfile(depends_exe) return depends_exe
def createNodeTree(filename): """ Create a node tree. Turn that source code into a node tree structure. If recursion into imported modules is available, more trees will be available during optimization, or immediately through recursed directory paths. """ # First, build the raw node tree from the source code. main_module = Building.buildModuleTree( filename=filename, package=None, is_top=True, is_main=not Options.shallMakeModule(), ) ModuleRegistry.addRootModule(main_module) # First remove old object files and old generated files, old binary or # module, and standalone mode program directory if any, they can only do # harm. source_dir = getSourceDirectoryPath(main_module) if not Options.shallOnlyExecCCompilerCall(): cleanSourceDirectory(source_dir) # Prepare the ".dist" directory, throwing away what was there before. if Options.isStandaloneMode(): standalone_dir = getStandaloneDirectoryPath(main_module) removeDirectory(path=standalone_dir, ignore_errors=True) makePath(standalone_dir) deleteFile(path=getResultFullpath(main_module), must_exist=False) # Second, do it for the directories given. for plugin_filename in Options.getShallFollowExtra(): Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None) for pattern in Options.getShallFollowExtraFilePatterns(): Recursion.checkPluginFilenamePattern(pattern=pattern) for package_name in Options.getMustIncludePackages(): package_package, package_directory, kind = Importing.findModule( importing=None, module_name=package_name, parent_package=None, level=0, warn=False, ) if kind != "absolute": sys.exit("Error, failed to locate package %r." % package_name) Recursion.checkPluginPath( plugin_filename=package_directory, module_package=package_package ) for module_name in Options.getMustIncludeModules(): module_package, module_filename, kind = Importing.findModule( importing=None, module_name=module_name, parent_package=None, level=0, warn=False, ) if kind != "absolute": sys.exit("Error, failed to locate module %r." % module_name) Recursion.checkPluginSinglePath( plugin_filename=module_filename, module_package=module_package ) # Then optimize the tree and potentially recursed modules. Optimization.optimize(main_module.getOutputFilename()) if Options.isExperimental("check_xml_persistence"): for module in ModuleRegistry.getRootModules(): if module.isMainModule(): return module assert False else: # Main module might change behind our back, look it up again. return main_module
def _detectBinaryPathDLLsWindows(original_dir, binary_filename, package_name): result = set() depends_exe = getDependsExePath() # The search order by default prefers the system directory, where a # wrong "PythonXX.dll" might be living. with open(binary_filename + ".dwp", 'w') as dwp_file: dwp_file.write("""\ KnownDLLs SysPath AppDir {original_dirs} 32BitSysDir 16BitSysDir OSDir AppPath SxS """.format( original_dirs = ( '\n'.join( "UserDir %s" % dirname for dirname in [original_dir] + getSubDirectories(original_dir) if not os.path.basename(dirname) == "__pycache__" if any(entry[1].lower().endswith(".dll") for entry in listDir(dirname)) ) if original_dir is not None else "" ) ) ) subprocess.call( ( depends_exe, "-c", "-ot%s" % binary_filename + ".depends", "-d:%s" % binary_filename + ".dwp", "-f1", "-pa1", "-ps1", binary_filename ), env = _makeBinaryPathPathDLLSearchEnv(package_name), ) inside = False first = False for line in open(binary_filename + ".depends"): if "| Module Dependency Tree |" in line: inside = True first = True continue if not inside: continue if "| Module List |" in line: break if ']' not in line: continue # Skip missing DLLs, apparently not needed anyway. if '?' in line[:line.find(']')]: continue # Skip DLLs that failed to load, apparently not needed anyway. if 'E' in line[:line.find(']')]: continue dll_filename = line[line.find(']')+2:-1] # The executable itself is of course exempted. We cannot check its path # because depends.exe mistreats unicode paths. if first: first = False continue assert os.path.isfile(dll_filename), dll_filename dll_name = os.path.basename(dll_filename).upper() # Win API can be assumed. if dll_name.startswith("API-MS-WIN-") or \ dll_name.startswith("EXT-MS-WIN-"): continue if dll_name in ("SHELL32.DLL", "USER32.DLL", "KERNEL32.DLL", "NTDLL.DLL", "NETUTILS.DLL", "LOGONCLI.DLL", "GDI32.DLL", "RPCRT4.DLL", "ADVAPI32.DLL", "SSPICLI.DLL", "SECUR32.DLL", "KERNELBASE.DLL", "WINBRAND.DLL", "DSROLE.DLL", "DNSAPI.DLL", "SAMCLI.DLL", "WKSCLI.DLL", "SAMLIB.DLL", "WLDAP32.DLL", "NTDSAPI.DLL", "CRYPTBASE.DLL", "W32TOPL", "WS2_32.DLL", "SPPC.DLL", "MSSIGN32.DLL", "CERTCLI.DLL", "WEBSERVICES.DLL", "AUTHZ.DLL", "CERTENROLL.DLL", "VAULTCLI.DLL", "REGAPI.DLL", "BROWCLI.DLL", "WINNSI.DLL", "DHCPCSVC6.DLL", "PCWUM.DLL", "CLBCATQ.DLL", "IMAGEHLP.DLL", "MSASN1.DLL", "DBGHELP.DLL", "DEVOBJ.DLL", "DRVSTORE.DLL", "CABINET.DLL", "SCECLI.DLL", "SPINF.DLL", "SPFILEQ.DLL", "GPAPI.DLL", "NETJOIN.DLL", "W32TOPL.DLL", "NETBIOS.DLL", "DXGI.DLL", "DWRITE.DLL", "D3D11.DLL", "WLANAPI.DLL", "WLANUTIL.DLL", "ONEX.DLL", "EAPPPRXY.DLL", "MFPLAT.DLL", "AVRT.DLL", "ELSCORE.DLL", "INETCOMM.DLL", "MSOERT2.DLL", "IEUI.DLL", "MSCTF.DLL", "MSFEEDS.DLL", "UIAUTOMATIONCORE.DLL", "PSAPI.DLL", "EFSADU.DLL", "MFC42U.DLL", "ODBC32.DLL", "OLEDLG.DLL", "NETAPI32.DLL", "LINKINFO.DLL", "DUI70.DLL", "ADVPACK.DLL", "NTSHRUI.DLL", "WINSPOOL.DRV", "EFSUTIL.DLL", "WINSCARD.DLL", "SHDOCVW.DLL", "IEFRAME.DLL", "D2D1.DLL", "GDIPLUS.DLL", "OCCACHE.DLL", "IEADVPACK.DLL", "MLANG.DLL", "MSI.DLL", "MSHTML.DLL", "COMDLG32.DLL", "PRINTUI.DLL", "PUIAPI.DLL", "ACLUI.DLL", "WTSAPI32.DLL", "FMS.DLL", "DFSCLI.DLL", "HLINK.DLL", "MSRATING.DLL", "PRNTVPT.DLL", "IMGUTIL.DLL", "MSLS31.DLL", "VERSION.DLL", "NORMALIZ.DLL", "IERTUTIL.DLL", "WININET.DLL", "WINTRUST.DLL", "XMLLITE.DLL", "APPHELP.DLL", "PROPSYS.DLL", "RSTRTMGR.DLL", "NCRYPT.DLL", "BCRYPT.DLL", "MMDEVAPI.DLL", "MSILTCFG.DLL", "DEVMGR.DLL", "DEVRTL.DLL", "NEWDEV.DLL", "VPNIKEAPI.DLL", "WINHTTP.DLL", "WEBIO.DLL", "NSI.DLL", "DHCPCSVC.DLL", "CRYPTUI.DLL", "ESENT.DLL", "DAVHLPR.DLL", "CSCAPI.DLL", "ATL.DLL", "OLEAUT32.DLL", "SRVCLI.DLL", "RASDLG.DLL", "MPRAPI.DLL", "RTUTILS.DLL", "RASMAN.DLL", "MPRMSG.DLL", "SLC.DLL", "CRYPTSP.DLL", "RASAPI32.DLL", "TAPI32.DLL", "EAPPCFG.DLL", "NDFAPI.DLL", "WDI.DLL", "COMCTL32.DLL", "UXTHEME.DLL", "IMM32.DLL", "OLEACC.DLL", "WINMM.DLL", "WINDOWSCODECS.DLL", "DWMAPI.DLL", "DUSER.DLL", "PROFAPI.DLL", "URLMON.DLL", "SHLWAPI.DLL", "LPK.DLL", "USP10.DLL", "CFGMGR32.DLL", "MSIMG32.DLL", "POWRPROF.DLL", "SETUPAPI.DLL", "WINSTA.DLL", "CRYPT32.DLL", "IPHLPAPI.DLL", "MPR.DLL", "CREDUI.DLL", "NETPLWIZ.DLL", "OLE32.DLL", "ACTIVEDS.DLL", "ADSLDPC.DLL", "USERENV.DLL", "APPREPAPI.DLL", "BCP47LANGS.DLL", "BCRYPTPRIMITIVES.DLL", "CERTCA.DLL", "CHARTV.DLL", "COMBASE.DLL", "COML2.DLL", "DCOMP.DLL", "DPAPI.DLL", "DSPARSE.DLL", "FECLIENT.DLL", "FIREWALLAPI.DLL", "FLTLIB.DLL", "MRMCORER.DLL", "NTASN1.DLL", "SECHOST.DLL", "SETTINGSYNCPOLICY.DLL", "SHCORE.DLL", "TBS.DLL", "TWINAPI.APPCORE.DLL", "TWINAPI.DLL", "VIRTDISK.DLL", "WEBSOCKET.DLL", "WEVTAPI.DLL", "WINMMBASE.DLL", "WMICLNT.DLL"): continue result.add( os.path.normcase(os.path.abspath(dll_filename)) ) deleteFile(binary_filename + ".depends", must_exist = True) deleteFile(binary_filename + ".dwp", must_exist = True) return result
def createNodeTree(filename): """Create a node tree. Turn that source code into a node tree structure. If recursion into imported modules is available, more trees will be available during optimization, or immediately through recursed directory paths. """ # First, build the raw node tree from the source code. main_module = Building.buildModuleTree( filename=filename, package=None, is_top=True, is_main=not Options.shallMakeModule(), ) # First remove old object files and old generated files, old binary or # module, and standalone mode program directory if any, they can only do # harm. source_dir = OutputDirectories.getSourceDirectoryPath() if not Options.shallOnlyExecCCompilerCall(): cleanSourceDirectory(source_dir) # Prepare the ".dist" directory, throwing away what was there before. if Options.isStandaloneMode(): standalone_dir = OutputDirectories.getStandaloneDirectoryPath() removeDirectory(path=standalone_dir, ignore_errors=True) makePath(standalone_dir) deleteFile(path=OutputDirectories.getResultFullpath(), must_exist=False) # Second, do it for the directories given. for plugin_filename in Options.getShallFollowExtra(): Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None) for pattern in Options.getShallFollowExtraFilePatterns(): Recursion.checkPluginFilenamePattern(pattern=pattern) for package_name in Options.getMustIncludePackages(): package_package, package_directory, kind = Importing.findModule( importing=None, module_name=ModuleName(package_name), parent_package=None, level=0, warn=False, ) if kind != "absolute": sys.exit("Error, failed to locate package %r." % package_name) Recursion.checkPluginPath( plugin_filename=package_directory, module_package=package_package ) for module_name in Options.getMustIncludeModules(): module_package, module_filename, kind = Importing.findModule( importing=None, module_name=ModuleName(module_name), parent_package=None, level=0, warn=False, ) if kind != "absolute": sys.exit("Error, failed to locate module %r." % module_name) Recursion.checkPluginSinglePath( plugin_filename=module_filename, module_package=module_package ) # Then optimize the tree and potentially recursed modules. Optimization.optimize(main_module.getOutputFilename()) if Options.isExperimental("check_xml_persistence"): for module in ModuleRegistry.getRootModules(): if module.isMainModule(): return module assert False else: # Main module might change behind our back, look it up again. return main_module
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir, binary_filename): """Pack to onefile binary on Linux. Notes: This is mostly a wrapper around AppImage, which does all the heavy lifting. """ if not locateDLL("fuse"): postprocessing_logger.sysexit("""\ Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed for onefile creation to work on Linux.""") # This might be possible to avoid being done with --runtime-file. apprun_filename = os.path.join(dist_dir, "AppRun") putTextFileContents( apprun_filename, contents="""\ #!/bin/bash exec -a $ARGV0 $APPDIR/%s \"$@\"""" % os.path.basename(binary_filename), ) addFileExecutablePermission(apprun_filename) binary_basename = os.path.basename(getResultBasepath()) icon_paths = getIconPaths() assert icon_paths extension = os.path.splitext(icon_paths[0])[1].lower() copyFile(icon_paths[0], getResultBasepath() + extension) putTextFileContents( getResultBasepath() + ".desktop", contents="""\ [Desktop Entry] Name=%(binary_basename)s Exec=%(binary_filename)s Icon=%(binary_basename)s Type=Application Categories=Utility;""" % { "binary_basename": binary_basename, "binary_filename": os.path.basename(binary_filename), }, ) postprocessing_logger.info( "Creating single file from dist folder, this may take a while.") stdout_filename = binary_filename + ".appimage.stdout.txt" stderr_filename = binary_filename + ".appimage.stderr.txt" stdout_file = openTextFile(stdout_filename, "wb") stderr_file = openTextFile(stderr_filename, "wb") command = ( _getAppImageToolPath(for_operation=True, assume_yes_for_downloads=assumeYesForDownloads()), dist_dir, "--comp", getAppImageCompression(), "-n", onefile_output_filename, ) stderr_file.write(b"Executed %r\n" % " ".join(command)) # Starting the process while locked, so file handles are not duplicated, we # need fine grained control over process here, therefore we cannot use the # Execution.executeProcess() function without making it too complex and not # all Python versions allow using with, pylint: disable=consider-using-with # pylint: disable appimagetool_process = subprocess.Popen( command, shell=False, stdin=getNullInput(), stdout=stdout_file, stderr=stderr_file, ) result = appimagetool_process.wait() stdout_file.close() stderr_file.close() if result != 0: # Useless result if there were errors, so now remove it. deleteFile(onefile_output_filename, must_exist=False) stderr = getFileContents(stderr_filename, mode="rb") if b"Text file busy" in stderr: postprocessing_logger.sysexit( "Error, error exit from AppImage because target file is locked." ) if b"modprobe fuse" in stderr: postprocessing_logger.sysexit( "Error, error exit from AppImage because fuse kernel module was not loaded." ) postprocessing_logger.sysexit( "Error, error exit from AppImage, check its outputs '%s' and '%s'." % (stdout_filename, stderr_filename)) if not os.path.exists(onefile_output_filename): postprocessing_logger.sysexit( "Error, expected output file %r not created by AppImage, check its outputs '%s' and '%s'." % (onefile_output_filename, stdout_filename, stderr_filename)) deleteFile(stdout_filename, must_exist=True) deleteFile(stderr_filename, must_exist=True) postprocessing_logger.info("Completed onefile creation.")
def getDependsExePath(): """ Return the path of depends.exe (for Windows). Will prompt the user to download if not already cached in AppData directory for Nuitka. """ if Utils.getArchitecture() == "x86": depends_url = "http://dependencywalker.com/depends22_x86.zip" else: depends_url = "http://dependencywalker.com/depends22_x64.zip" if "APPDATA" not in os.environ: sys.exit("Error, standalone mode cannot find 'APPDATA' environment.") nuitka_app_dir = os.path.join(os.environ["APPDATA"], "nuitka") if not os.path.isdir(nuitka_app_dir): makePath(nuitka_app_dir) nuitka_depends_zip = os.path.join(nuitka_app_dir, os.path.basename(depends_url)) if not os.path.isfile(nuitka_depends_zip): Tracing.printLine("""\ Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool to analyze the dependencies of Python extension modules. Is it OK to download and put it in APPDATA (no installer needed, cached, one time question).""") reply = raw_input("Proceed and download? [Yes]/No ") if reply.lower() in ("no", 'n'): sys.exit( "Nuitka does not work in --standalone on Windows without.") info("Downloading '%s'" % depends_url) urlretrieve(depends_url, nuitka_depends_zip) nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture()) if not os.path.isdir(nuitka_depends_dir): os.makedirs(nuitka_depends_dir) depends_exe = os.path.join(nuitka_depends_dir, "depends.exe") if not os.path.isfile(depends_exe): info("Extracting to '%s'" % depends_exe) import zipfile try: depends_zip = zipfile.ZipFile(nuitka_depends_zip) depends_zip.extractall(nuitka_depends_dir) except Exception: # Catching anything zip throws, pylint:disable=W0703 info("Problem with the downloaded zip file, deleting it.") deleteFile(depends_exe, must_exist=False) deleteFile(nuitka_depends_zip, must_exist=True) sys.exit("Error, need '%s' as extracted from '%s'." % (depends_exe, depends_url)) assert os.path.isfile(depends_exe) return depends_exe
def check(path): if hasFilenameExtension(path, extensions): deleteFile(path, must_exist=True)
def compileAndCompareWith(nuitka, pass_number): if "PYTHONHASHSEED" not in os.environ: os.environ["PYTHONHASHSEED"] = "0" base_dir = os.path.join("..", "..") for package in PACKAGE_LIST: package = package.replace("/", os.path.sep) source_dir = os.path.join(base_dir, package) for path, filename in listDir(source_dir): if not filename.endswith(".py"): continue if filename.startswith(".#"): continue path = os.path.join(source_dir, filename) if filename != "__init__.py": _traceCompilation(path=path, pass_number=pass_number) target = filename.replace(".py", ".build") target_dir = os.path.join(tmp_dir, target) removeDirectory(path=target_dir, ignore_errors=False) command = [ nuitka, "--module", "--plugin-enable=pylint-warnings", "--output-dir=%s" % tmp_dir, "--no-pyi-file", "--nofollow-imports", path, ] command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split() my_print("Command: ", " ".join(command)) exit_nuitka = subprocess.call(command) # In case of segfault or assertion triggered, run in debugger. if exit_nuitka in (-11, -6) and sys.platform != "nt": command2 = wrapCommandForDebuggerForSubprocess(*command) subprocess.call(command2) if exit_nuitka != 0: my_print("An error exit %s occurred, aborting." % exit_nuitka) sys.exit(exit_nuitka) has_diff = diffRecursive(os.path.join(package, target), target_dir) if has_diff: sys.exit("There were differences!") shutil.rmtree(target_dir) for preferred in (True, False): target_filename = filename.replace( ".py", getSharedLibrarySuffix(preferred=preferred)) deleteFile(path=os.path.join(tmp_dir, target_filename), must_exist=False)