def detectDLLsWithDependencyWalker(binary_filename, scan_dirs):
dwp_filename = binary_filename + ".dwp"
output_filename = binary_filename + ".depends"
# User query should only happen once if at all.
with withFileLock(
"Finding out dependency walker path and creating DWP file for %s"
% binary_filename
):
depends_exe = getDependsExePath()
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
putTextFileContents(
dwp_filename,
contents="""\
%(scan_dirs)s
SxS
"""
% {
"scan_dirs": "\n".join(
"UserDir %s" % getExternalUsePath(dirname) for dirname in scan_dirs
)
},
)
# Starting the process while locked, so file handles are not duplicated.
# TODO: At least exit code should be checked, output goes to a filename,
# but errors might be interesting potentially.
_stdout, _stderr, _exit_code = executeProcess(
command=(
depends_exe,
"-c",
"-ot%s" % output_filename,
"-d:%s" % dwp_filename,
"-f1",
"-pa1",
"-ps1",
binary_filename,
),
external_cwd=True,
)
if not os.path.exists(output_filename):
inclusion_logger.sysexit(
"Error, 'depends.exe' failed to produce expected output."
)
# Opening the result under lock, so it is not getting locked by new processes.
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
result = parseDependsExeOutput(output_filename)
deleteFile(output_filename, must_exist=True)
deleteFile(dwp_filename, must_exist=True)
return result
def detectDLLsWithDependencyWalker(binary_filename, scan_dirs):
dwp_filename = binary_filename + ".dwp"
output_filename = binary_filename + ".depends"
# User query should only happen once if at all.
with withFileLock(
"Finding out dependency walker path and creating DWP file for %s" %
binary_filename):
depends_exe = getDependsExePath()
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
with open(dwp_filename, "w") as dwp_file:
dwp_file.write(
"""\
%(scan_dirs)s
SxS
""" % {
"scan_dirs":
"\n".join("UserDir %s" % getExternalUsePath(dirname)
for dirname in scan_dirs)
})
# Starting the process while locked, so file handles are not duplicated.
depends_exe_process = subprocess.Popen(
(
depends_exe,
"-c",
"-ot%s" % output_filename,
"-d:%s" % dwp_filename,
"-f1",
"-pa1",
"-ps1",
binary_filename,
),
stdin=getNullInput(),
cwd=getExternalUsePath(os.getcwd()),
)
# TODO: Exit code should be checked.
depends_exe_process.wait()
if not os.path.exists(output_filename):
inclusion_logger.sysexit(
"Error, depends.exe failed to produce expected output.")
# Opening the result under lock, so it is not getting locked by new processes.
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
result = _parseDependsExeOutput(output_filename)
deleteFile(output_filename, must_exist=True)
deleteFile(dwp_filename, must_exist=True)
return result
def cleanSourceDirectory(source_dir):
extensions = (".bin", ".c", ".cpp", ".exp", ".h", ".lib", ".manifest",
".o", ".obj", ".os", ".rc", ".res", ".S")
if os.path.isdir(source_dir):
for path, _filename in listDir(source_dir):
if hasFilenameExtension(path, extensions):
deleteFile(path, must_exist=True)
else:
makePath(source_dir)
def _deleteMsvcPGOFiles(pgo_mode):
assert _wasMsvcMode()
msvc_pgc_filename = OutputDirectories.getResultBasepath(onefile=False) + "!1.pgc"
deleteFile(msvc_pgc_filename, must_exist=False)
if pgo_mode == "use":
msvc_pgd_filename = OutputDirectories.getResultBasepath(onefile=False) + ".pgd"
deleteFile(msvc_pgd_filename, must_exist=False)
return msvc_pgc_filename
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule())
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename,
module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize()
if Options.isExperimental():
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def cleanSourceDirectory(source_dir):
extensions = (
".bin",
".c",
".cpp",
".exp",
".h",
".lib",
".manifest",
".o",
".obj",
".os",
".rc",
".res",
".S",
".txt",
)
if os.path.isdir(source_dir):
for path, _filename in listDir(source_dir):
if hasFilenameExtension(path, extensions):
deleteFile(path, must_exist=True)
else:
makePath(source_dir)
def _createNodeTree(filename):
"""Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# Many cases to deal with, pylint: disable=too-many-branches
# First, build the raw node tree from the source code.
main_module = Building.buildMainModuleTree(
filename=filename,
is_main=not Options.shallMakeModule(),
)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
SconsInterface.cleanSconsDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = OutputDirectories.getStandaloneDirectoryPath(bundle=False)
removeDirectory(path=standalone_dir, ignore_errors=True)
if Options.shallCreateAppBundle():
removeDirectory(
path=changeFilenameExtension(standalone_dir, ".app"), ignore_errors=True
)
# Delete result file, to avoid confusion with previous build and to
# avoid locking issues after the build.
deleteFile(
path=OutputDirectories.getResultFullpath(onefile=False), must_exist=False
)
if Options.isOnefileMode():
deleteFile(
path=OutputDirectories.getResultFullpath(onefile=True), must_exist=False
)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_name, package_directory, kind = Importing.locateModule(
module_name=ModuleName(package_name),
parent_package=None,
level=0,
)
if kind != "absolute":
inclusion_logger.sysexit(
"Error, failed to locate package %r you asked to include."
% package_name.asString()
)
Recursion.checkPluginPath(
plugin_filename=package_directory,
module_package=package_name.getPackageName(),
)
for module_name in Options.getMustIncludeModules():
module_name, module_filename, kind = Importing.locateModule(
module_name=ModuleName(module_name),
parent_package=None,
level=0,
)
if kind != "absolute":
inclusion_logger.sysexit(
"Error, failed to locate module '%s' you asked to include."
% module_name.asString()
)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_name.getPackageName()
)
# Allow plugins to add more modules based on the initial set being complete.
Plugins.onModuleInitialSet()
# Then optimize the tree and potentially recursed modules.
optimizeModules(main_module.getOutputFilename())
# Allow plugins to comment on final module set.
Plugins.onModuleCompleteSet()
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def _detectBinaryPathDLLsWindows(is_main_executable, source_dir, original_dir,
binary_filename, package_name):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-locals
result = set()
cache_filename = _getCacheFilename(is_main_executable, source_dir,
original_dir, binary_filename)
if os.path.exists(cache_filename
) and not Options.shallNotUseDependsExeCachedResults():
for line in open(cache_filename):
line = line.strip()
result.add(line)
return result
# User query should only happen once if at all.
with _withLock():
depends_exe = getDependsExePath()
scan_dirs = [sys.prefix]
if package_name is not None:
from nuitka.importing.Importing import findModule
package_dir = findModule(None, package_name, None, 0, False)[1]
if os.path.isdir(package_dir):
scan_dirs.append(package_dir)
scan_dirs.extend(getSubDirectories(package_dir))
if original_dir is not None:
scan_dirs.append(original_dir)
scan_dirs.extend(getSubDirectories(original_dir))
with _withLock():
# The search order by default prefers the system directory, where a
# wrong "PythonXX.dll" might be living.
with open(binary_filename + ".dwp", 'w') as dwp_file:
dwp_file.write(
"""\
KnownDLLs
%(original_dirs)s
SysPath
32BitSysDir
16BitSysDir
OSDir
AppPath
SxS
""" % {
"original_dirs":
'\n'.join("UserDir %s" % dirname for dirname in scan_dirs
if not os.path.basename(dirname) == "__pycache__"
if any(entry[1].lower().endswith(".dll")
for entry in listDir(dirname))),
})
# Starting the process while locked, so file handles are not duplicated.
depends_exe_process = subprocess.Popen(
(depends_exe, "-c", "-ot%s" % binary_filename + ".depends",
"-d:%s" % binary_filename + ".dwp", "-f1", "-pa1", "-ps1",
binary_filename))
# TODO: Exit code should be checked.
depends_exe_process.wait()
# Opening the result under lock, so it is not getting locked by new processes.
with _withLock():
_parseDependsExeOutput(binary_filename + ".depends", result)
deleteFile(binary_filename + ".depends", must_exist=True)
deleteFile(binary_filename + ".dwp", must_exist=True)
if not Options.shallNotStoreDependsExeCachedResults():
with open(cache_filename, 'w') as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def getDependsExePath():
""" Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if Utils.getArchitecture() == "x86":
depends_url = "http://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "http://dependencywalker.com/depends22_x64.zip"
nuitka_app_dir = getAppDir()
nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture())
nuitka_depends_zip = os.path.join(nuitka_depends_dir,
os.path.basename(depends_url))
depends_exe = os.path.join(nuitka_depends_dir, "depends.exe")
makePath(nuitka_depends_dir)
if not os.path.isfile(nuitka_depends_zip) and not os.path.isfile(
depends_exe):
if assumeYesForDownloads():
reply = "y"
else:
Tracing.printLine("""\
Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool
to analyze the dependencies of Python extension modules. Is it OK to download
and put it in "%s".
No installer needed, cached, one time question.
Proceed and download? [Yes]/No """ % (nuitka_app_dir))
Tracing.flushStdout()
reply = raw_input()
if reply.lower() in ("no", "n"):
sys.exit(
"Nuitka does not work in --standalone on Windows without.")
info("Downloading '%s'" % depends_url)
try:
urlretrieve(depends_url, nuitka_depends_zip)
except Exception: # Any kind of error, pylint: disable=broad-except
sys.exit("""Failed to download '%s'.\
Contents should manually be extracted to '%s'.""" %
(depends_url, nuitka_depends_dir))
if not os.path.isfile(depends_exe):
info("Extracting to '%s'" % depends_exe)
import zipfile
try:
depends_zip = zipfile.ZipFile(nuitka_depends_zip)
depends_zip.extractall(nuitka_depends_dir)
except Exception: # Catching anything zip throws, pylint: disable=broad-except
info("Problem with the downloaded zip file, deleting it.")
deleteFile(depends_exe, must_exist=False)
deleteFile(nuitka_depends_zip, must_exist=True)
sys.exit("Error, need '%s' as extracted from '%s'." %
(depends_exe, depends_url))
assert os.path.isfile(depends_exe)
return depends_exe
def detectBinaryPathDLLsWindowsDependencyWalker(
is_main_executable,
source_dir,
original_dir,
binary_filename,
package_name,
use_cache,
update_cache,
):
# This is complex, as it also includes the caching mechanism
# pylint: disable=too-many-locals
result = set()
if use_cache or update_cache:
cache_filename = _getCacheFilename(
dependency_tool="depends.exe",
is_main_executable=is_main_executable,
source_dir=source_dir,
original_dir=original_dir,
binary_filename=binary_filename,
)
if use_cache:
with withFileLock():
if not os.path.exists(cache_filename):
use_cache = False
if use_cache:
for line in getFileContentByLine(cache_filename):
line = line.strip()
result.add(line)
return result
if Options.isShowProgress():
info("Analysing dependencies of '%s'." % binary_filename)
scan_dirs = getScanDirectories(package_name, original_dir)
dwp_filename = binary_filename + ".dwp"
output_filename = binary_filename + ".depends"
# User query should only happen once if at all.
with _withLock(
"Finding out dependency walker path and creating DWP file for %s"
% binary_filename
):
depends_exe = getDependsExePath()
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
with open(dwp_filename, "w") as dwp_file:
dwp_file.write(
"""\
%(scan_dirs)s
SxS
"""
% {
"scan_dirs": "\n".join(
"UserDir %s" % dirname for dirname in scan_dirs
)
}
)
# Starting the process while locked, so file handles are not duplicated.
depends_exe_process = subprocess.Popen(
(
depends_exe,
"-c",
"-ot%s" % output_filename,
"-d:%s" % dwp_filename,
"-f1",
"-pa1",
"-ps1",
binary_filename,
),
cwd=getExternalUsePath(os.getcwd()),
)
# TODO: Exit code should be checked.
depends_exe_process.wait()
if not os.path.exists(output_filename):
sys.exit("Error, depends.exe failed to product output.")
# Opening the result under lock, so it is not getting locked by new processes.
# Note: Do this under lock to avoid forked processes to hold
# a copy of the file handle on Windows.
_parseDependsExeOutput(output_filename, result)
deleteFile(output_filename, must_exist=True)
deleteFile(dwp_filename, must_exist=True)
if update_cache:
with open(cache_filename, "w") as cache_file:
for dll_filename in result:
print(dll_filename, file=cache_file)
return result
def getDependsExePath():
""" Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if Utils.getArchitecture() == "x86":
depends_url = "http://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "http://dependencywalker.com/depends22_x64.zip"
nuitka_app_dir = getAppDir()
nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture())
nuitka_depends_zip = os.path.join(nuitka_depends_dir, os.path.basename(depends_url))
depends_exe = os.path.join(nuitka_depends_dir, "depends.exe")
makePath(nuitka_depends_dir)
if not os.path.isfile(nuitka_depends_zip) and not os.path.isfile(depends_exe):
if assumeYesForDownloads():
reply = "y"
else:
Tracing.printLine(
"""\
Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool
to analyze the dependencies of Python extension modules. Is it OK to download
and put it in "%s".
No installer needed, cached, one time question.
Proceed and download? [Yes]/No """
% (nuitka_app_dir)
)
Tracing.flushStdout()
reply = raw_input()
if reply.lower() in ("no", "n"):
sys.exit("Nuitka does not work in --standalone on Windows without.")
info("Downloading '%s'" % depends_url)
try:
urlretrieve(depends_url, nuitka_depends_zip)
except Exception: # Any kind of error, pylint: disable=broad-except
sys.exit(
"""Failed to download '%s'.\
Contents should manually be extracted to '%s'."""
% (depends_url, nuitka_depends_dir)
)
if not os.path.isfile(depends_exe):
info("Extracting to '%s'" % depends_exe)
import zipfile
try:
depends_zip = zipfile.ZipFile(nuitka_depends_zip)
depends_zip.extractall(nuitka_depends_dir)
except Exception: # Catching anything zip throws, pylint:disable=W0703
info("Problem with the downloaded zip file, deleting it.")
deleteFile(depends_exe, must_exist=False)
deleteFile(nuitka_depends_zip, must_exist=True)
sys.exit(
"Error, need '%s' as extracted from '%s'." % (depends_exe, depends_url)
)
assert os.path.isfile(depends_exe)
return depends_exe
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule(),
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCCompilerCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing=None,
module_name=package_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate package %r." % package_name)
Recursion.checkPluginPath(
plugin_filename=package_directory, module_package=package_package
)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing=None,
module_name=module_name,
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate module %r." % module_name)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_package
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def _detectBinaryPathDLLsWindows(original_dir, binary_filename, package_name):
result = set()
depends_exe = getDependsExePath()
# The search order by default prefers the system directory, where a
# wrong "PythonXX.dll" might be living.
with open(binary_filename + ".dwp", 'w') as dwp_file:
dwp_file.write("""\
KnownDLLs
SysPath
AppDir
{original_dirs}
32BitSysDir
16BitSysDir
OSDir
AppPath
SxS
""".format(
original_dirs = (
'\n'.join(
"UserDir %s" % dirname
for dirname in
[original_dir] + getSubDirectories(original_dir)
if not os.path.basename(dirname) == "__pycache__"
if any(entry[1].lower().endswith(".dll") for entry in listDir(dirname))
)
if original_dir is not None
else ""
)
)
)
subprocess.call(
(
depends_exe,
"-c",
"-ot%s" % binary_filename + ".depends",
"-d:%s" % binary_filename + ".dwp",
"-f1",
"-pa1",
"-ps1",
binary_filename
),
env = _makeBinaryPathPathDLLSearchEnv(package_name),
)
inside = False
first = False
for line in open(binary_filename + ".depends"):
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if ']' not in line:
continue
# Skip missing DLLs, apparently not needed anyway.
if '?' in line[:line.find(']')]:
continue
# Skip DLLs that failed to load, apparently not needed anyway.
if 'E' in line[:line.find(']')]:
continue
dll_filename = line[line.find(']')+2:-1]
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert os.path.isfile(dll_filename), dll_filename
dll_name = os.path.basename(dll_filename).upper()
# Win API can be assumed.
if dll_name.startswith("API-MS-WIN-") or \
dll_name.startswith("EXT-MS-WIN-"):
continue
if dll_name in ("SHELL32.DLL", "USER32.DLL", "KERNEL32.DLL",
"NTDLL.DLL", "NETUTILS.DLL", "LOGONCLI.DLL", "GDI32.DLL",
"RPCRT4.DLL", "ADVAPI32.DLL", "SSPICLI.DLL", "SECUR32.DLL",
"KERNELBASE.DLL", "WINBRAND.DLL", "DSROLE.DLL", "DNSAPI.DLL",
"SAMCLI.DLL", "WKSCLI.DLL", "SAMLIB.DLL", "WLDAP32.DLL",
"NTDSAPI.DLL", "CRYPTBASE.DLL", "W32TOPL", "WS2_32.DLL",
"SPPC.DLL", "MSSIGN32.DLL", "CERTCLI.DLL", "WEBSERVICES.DLL",
"AUTHZ.DLL", "CERTENROLL.DLL", "VAULTCLI.DLL", "REGAPI.DLL",
"BROWCLI.DLL", "WINNSI.DLL", "DHCPCSVC6.DLL", "PCWUM.DLL",
"CLBCATQ.DLL", "IMAGEHLP.DLL", "MSASN1.DLL", "DBGHELP.DLL",
"DEVOBJ.DLL", "DRVSTORE.DLL", "CABINET.DLL", "SCECLI.DLL",
"SPINF.DLL", "SPFILEQ.DLL", "GPAPI.DLL", "NETJOIN.DLL",
"W32TOPL.DLL", "NETBIOS.DLL", "DXGI.DLL", "DWRITE.DLL",
"D3D11.DLL", "WLANAPI.DLL", "WLANUTIL.DLL", "ONEX.DLL",
"EAPPPRXY.DLL", "MFPLAT.DLL", "AVRT.DLL", "ELSCORE.DLL",
"INETCOMM.DLL", "MSOERT2.DLL", "IEUI.DLL", "MSCTF.DLL",
"MSFEEDS.DLL", "UIAUTOMATIONCORE.DLL", "PSAPI.DLL",
"EFSADU.DLL", "MFC42U.DLL", "ODBC32.DLL", "OLEDLG.DLL",
"NETAPI32.DLL", "LINKINFO.DLL", "DUI70.DLL", "ADVPACK.DLL",
"NTSHRUI.DLL", "WINSPOOL.DRV", "EFSUTIL.DLL", "WINSCARD.DLL",
"SHDOCVW.DLL", "IEFRAME.DLL", "D2D1.DLL", "GDIPLUS.DLL",
"OCCACHE.DLL", "IEADVPACK.DLL", "MLANG.DLL", "MSI.DLL",
"MSHTML.DLL", "COMDLG32.DLL", "PRINTUI.DLL", "PUIAPI.DLL",
"ACLUI.DLL", "WTSAPI32.DLL", "FMS.DLL", "DFSCLI.DLL",
"HLINK.DLL", "MSRATING.DLL", "PRNTVPT.DLL", "IMGUTIL.DLL",
"MSLS31.DLL", "VERSION.DLL", "NORMALIZ.DLL", "IERTUTIL.DLL",
"WININET.DLL", "WINTRUST.DLL", "XMLLITE.DLL", "APPHELP.DLL",
"PROPSYS.DLL", "RSTRTMGR.DLL", "NCRYPT.DLL", "BCRYPT.DLL",
"MMDEVAPI.DLL", "MSILTCFG.DLL", "DEVMGR.DLL", "DEVRTL.DLL",
"NEWDEV.DLL", "VPNIKEAPI.DLL", "WINHTTP.DLL", "WEBIO.DLL",
"NSI.DLL", "DHCPCSVC.DLL", "CRYPTUI.DLL", "ESENT.DLL",
"DAVHLPR.DLL", "CSCAPI.DLL", "ATL.DLL", "OLEAUT32.DLL",
"SRVCLI.DLL", "RASDLG.DLL", "MPRAPI.DLL", "RTUTILS.DLL",
"RASMAN.DLL", "MPRMSG.DLL", "SLC.DLL", "CRYPTSP.DLL",
"RASAPI32.DLL", "TAPI32.DLL", "EAPPCFG.DLL", "NDFAPI.DLL",
"WDI.DLL", "COMCTL32.DLL", "UXTHEME.DLL", "IMM32.DLL",
"OLEACC.DLL", "WINMM.DLL", "WINDOWSCODECS.DLL", "DWMAPI.DLL",
"DUSER.DLL", "PROFAPI.DLL", "URLMON.DLL", "SHLWAPI.DLL",
"LPK.DLL", "USP10.DLL", "CFGMGR32.DLL", "MSIMG32.DLL",
"POWRPROF.DLL", "SETUPAPI.DLL", "WINSTA.DLL", "CRYPT32.DLL",
"IPHLPAPI.DLL", "MPR.DLL", "CREDUI.DLL", "NETPLWIZ.DLL",
"OLE32.DLL", "ACTIVEDS.DLL", "ADSLDPC.DLL", "USERENV.DLL",
"APPREPAPI.DLL", "BCP47LANGS.DLL", "BCRYPTPRIMITIVES.DLL",
"CERTCA.DLL", "CHARTV.DLL", "COMBASE.DLL", "COML2.DLL",
"DCOMP.DLL", "DPAPI.DLL", "DSPARSE.DLL", "FECLIENT.DLL",
"FIREWALLAPI.DLL", "FLTLIB.DLL", "MRMCORER.DLL", "NTASN1.DLL",
"SECHOST.DLL", "SETTINGSYNCPOLICY.DLL", "SHCORE.DLL", "TBS.DLL",
"TWINAPI.APPCORE.DLL", "TWINAPI.DLL", "VIRTDISK.DLL",
"WEBSOCKET.DLL", "WEVTAPI.DLL", "WINMMBASE.DLL", "WMICLNT.DLL"):
continue
result.add(
os.path.normcase(os.path.abspath(dll_filename))
)
deleteFile(binary_filename + ".depends", must_exist = True)
deleteFile(binary_filename + ".dwp", must_exist = True)
return result
def createNodeTree(filename):
"""Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule(),
)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = OutputDirectories.getSourceDirectoryPath()
if not Options.shallOnlyExecCCompilerCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = OutputDirectories.getStandaloneDirectoryPath()
removeDirectory(path=standalone_dir, ignore_errors=True)
makePath(standalone_dir)
deleteFile(path=OutputDirectories.getResultFullpath(), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename, module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
for package_name in Options.getMustIncludePackages():
package_package, package_directory, kind = Importing.findModule(
importing=None,
module_name=ModuleName(package_name),
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate package %r." % package_name)
Recursion.checkPluginPath(
plugin_filename=package_directory, module_package=package_package
)
for module_name in Options.getMustIncludeModules():
module_package, module_filename, kind = Importing.findModule(
importing=None,
module_name=ModuleName(module_name),
parent_package=None,
level=0,
warn=False,
)
if kind != "absolute":
sys.exit("Error, failed to locate module %r." % module_name)
Recursion.checkPluginSinglePath(
plugin_filename=module_filename, module_package=module_package
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize(main_module.getOutputFilename())
if Options.isExperimental("check_xml_persistence"):
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def packDistFolderToOnefileLinux(onefile_output_filename, dist_dir,
binary_filename):
"""Pack to onefile binary on Linux.
Notes: This is mostly a wrapper around AppImage, which does all the heavy
lifting.
"""
if not locateDLL("fuse"):
postprocessing_logger.sysexit("""\
Error, the fuse library (libfuse.so.x from fuse2, *not* fuse3) must be installed
for onefile creation to work on Linux.""")
# This might be possible to avoid being done with --runtime-file.
apprun_filename = os.path.join(dist_dir, "AppRun")
putTextFileContents(
apprun_filename,
contents="""\
#!/bin/bash
exec -a $ARGV0 $APPDIR/%s \"$@\"""" % os.path.basename(binary_filename),
)
addFileExecutablePermission(apprun_filename)
binary_basename = os.path.basename(getResultBasepath())
icon_paths = getIconPaths()
assert icon_paths
extension = os.path.splitext(icon_paths[0])[1].lower()
copyFile(icon_paths[0], getResultBasepath() + extension)
putTextFileContents(
getResultBasepath() + ".desktop",
contents="""\
[Desktop Entry]
Name=%(binary_basename)s
Exec=%(binary_filename)s
Icon=%(binary_basename)s
Type=Application
Categories=Utility;""" % {
"binary_basename": binary_basename,
"binary_filename": os.path.basename(binary_filename),
},
)
postprocessing_logger.info(
"Creating single file from dist folder, this may take a while.")
stdout_filename = binary_filename + ".appimage.stdout.txt"
stderr_filename = binary_filename + ".appimage.stderr.txt"
stdout_file = openTextFile(stdout_filename, "wb")
stderr_file = openTextFile(stderr_filename, "wb")
command = (
_getAppImageToolPath(for_operation=True,
assume_yes_for_downloads=assumeYesForDownloads()),
dist_dir,
"--comp",
getAppImageCompression(),
"-n",
onefile_output_filename,
)
stderr_file.write(b"Executed %r\n" % " ".join(command))
# Starting the process while locked, so file handles are not duplicated, we
# need fine grained control over process here, therefore we cannot use the
# Execution.executeProcess() function without making it too complex and not
# all Python versions allow using with, pylint: disable=consider-using-with
# pylint: disable
appimagetool_process = subprocess.Popen(
command,
shell=False,
stdin=getNullInput(),
stdout=stdout_file,
stderr=stderr_file,
)
result = appimagetool_process.wait()
stdout_file.close()
stderr_file.close()
if result != 0:
# Useless result if there were errors, so now remove it.
deleteFile(onefile_output_filename, must_exist=False)
stderr = getFileContents(stderr_filename, mode="rb")
if b"Text file busy" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because target file is locked."
)
if b"modprobe fuse" in stderr:
postprocessing_logger.sysexit(
"Error, error exit from AppImage because fuse kernel module was not loaded."
)
postprocessing_logger.sysexit(
"Error, error exit from AppImage, check its outputs '%s' and '%s'."
% (stdout_filename, stderr_filename))
if not os.path.exists(onefile_output_filename):
postprocessing_logger.sysexit(
"Error, expected output file %r not created by AppImage, check its outputs '%s' and '%s'."
% (onefile_output_filename, stdout_filename, stderr_filename))
deleteFile(stdout_filename, must_exist=True)
deleteFile(stderr_filename, must_exist=True)
postprocessing_logger.info("Completed onefile creation.")
def getDependsExePath():
""" Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if Utils.getArchitecture() == "x86":
depends_url = "http://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "http://dependencywalker.com/depends22_x64.zip"
if "APPDATA" not in os.environ:
sys.exit("Error, standalone mode cannot find 'APPDATA' environment.")
nuitka_app_dir = os.path.join(os.environ["APPDATA"], "nuitka")
if not os.path.isdir(nuitka_app_dir):
makePath(nuitka_app_dir)
nuitka_depends_zip = os.path.join(nuitka_app_dir,
os.path.basename(depends_url))
if not os.path.isfile(nuitka_depends_zip):
Tracing.printLine("""\
Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool
to analyze the dependencies of Python extension modules. Is it OK to download
and put it in APPDATA (no installer needed, cached, one time question).""")
reply = raw_input("Proceed and download? [Yes]/No ")
if reply.lower() in ("no", 'n'):
sys.exit(
"Nuitka does not work in --standalone on Windows without.")
info("Downloading '%s'" % depends_url)
urlretrieve(depends_url, nuitka_depends_zip)
nuitka_depends_dir = os.path.join(nuitka_app_dir, Utils.getArchitecture())
if not os.path.isdir(nuitka_depends_dir):
os.makedirs(nuitka_depends_dir)
depends_exe = os.path.join(nuitka_depends_dir, "depends.exe")
if not os.path.isfile(depends_exe):
info("Extracting to '%s'" % depends_exe)
import zipfile
try:
depends_zip = zipfile.ZipFile(nuitka_depends_zip)
depends_zip.extractall(nuitka_depends_dir)
except Exception: # Catching anything zip throws, pylint:disable=W0703
info("Problem with the downloaded zip file, deleting it.")
deleteFile(depends_exe, must_exist=False)
deleteFile(nuitka_depends_zip, must_exist=True)
sys.exit("Error, need '%s' as extracted from '%s'." %
(depends_exe, depends_url))
assert os.path.isfile(depends_exe)
return depends_exe
def check(path):
if hasFilenameExtension(path, extensions):
deleteFile(path, must_exist=True)
def compileAndCompareWith(nuitka, pass_number):
if "PYTHONHASHSEED" not in os.environ:
os.environ["PYTHONHASHSEED"] = "0"
base_dir = os.path.join("..", "..")
for package in PACKAGE_LIST:
package = package.replace("/", os.path.sep)
source_dir = os.path.join(base_dir, package)
for path, filename in listDir(source_dir):
if not filename.endswith(".py"):
continue
if filename.startswith(".#"):
continue
path = os.path.join(source_dir, filename)
if filename != "__init__.py":
_traceCompilation(path=path, pass_number=pass_number)
target = filename.replace(".py", ".build")
target_dir = os.path.join(tmp_dir, target)
removeDirectory(path=target_dir, ignore_errors=False)
command = [
nuitka,
"--module",
"--plugin-enable=pylint-warnings",
"--output-dir=%s" % tmp_dir,
"--no-pyi-file",
"--nofollow-imports",
path,
]
command += os.environ.get("NUITKA_EXTRA_OPTIONS", "").split()
my_print("Command: ", " ".join(command))
exit_nuitka = subprocess.call(command)
# In case of segfault or assertion triggered, run in debugger.
if exit_nuitka in (-11, -6) and sys.platform != "nt":
command2 = wrapCommandForDebuggerForSubprocess(*command)
subprocess.call(command2)
if exit_nuitka != 0:
my_print("An error exit %s occurred, aborting." %
exit_nuitka)
sys.exit(exit_nuitka)
has_diff = diffRecursive(os.path.join(package, target),
target_dir)
if has_diff:
sys.exit("There were differences!")
shutil.rmtree(target_dir)
for preferred in (True, False):
target_filename = filename.replace(
".py", getSharedLibrarySuffix(preferred=preferred))
deleteFile(path=os.path.join(tmp_dir, target_filename),
must_exist=False)
