def cleanSourceDirectory(source_dir):
if Utils.isDir(source_dir):
for path, _filename in Utils.listDir(source_dir):
if Utils.getExtension(path) in (".c", ".h", ".o", ".os", ".obj",
".bin", ".res", ".rc", ".S",
".cpp", ".manifest"):
Utils.deleteFile(path, True)
else:
Utils.makePath(source_dir)
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename = filename,
package = None,
is_top = True,
is_main = not Options.shallMakeModule()
)
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
shutil.rmtree(standalone_dir, ignore_errors = True)
Utils.makePath(standalone_dir)
Utils.deleteFile(
path = getResultFullpath(main_module),
must_exist = False
)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(
plugin_filename = plugin_filename,
module_package = None
)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(
pattern = pattern
)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize()
return main_module
def createNodeTree(filename):
""" Create a node tree.
Turn that source code into a node tree structure. If recursion into
imported modules is available, more trees will be available during
optimization, or immediately through recursed directory paths.
"""
# First, build the raw node tree from the source code.
main_module = Building.buildModuleTree(
filename=filename,
package=None,
is_top=True,
is_main=not Options.shallMakeModule())
ModuleRegistry.addRootModule(main_module)
# First remove old object files and old generated files, old binary or
# module, and standalone mode program directory if any, they can only do
# harm.
source_dir = getSourceDirectoryPath(main_module)
if not Options.shallOnlyExecCppCall():
cleanSourceDirectory(source_dir)
# Prepare the ".dist" directory, throwing away what was there before.
if Options.isStandaloneMode():
standalone_dir = getStandaloneDirectoryPath(main_module)
shutil.rmtree(standalone_dir, ignore_errors=True)
Utils.makePath(standalone_dir)
Utils.deleteFile(path=getResultFullpath(main_module), must_exist=False)
# Second, do it for the directories given.
for plugin_filename in Options.getShallFollowExtra():
Recursion.checkPluginPath(plugin_filename=plugin_filename,
module_package=None)
for pattern in Options.getShallFollowExtraFilePatterns():
Recursion.checkPluginFilenamePattern(pattern=pattern)
# Then optimize the tree and potentially recursed modules.
Optimization.optimize()
if Options.isExperimental():
for module in ModuleRegistry.getRootModules():
if module.isMainModule():
return module
assert False
else:
# Main module might change behind our back, look it up again.
return main_module
def cleanSourceDirectory(source_dir):
if Utils.isDir(source_dir):
for path, _filename in Utils.listDir(source_dir):
if Utils.getExtension(path) in (
".c",
".h",
".o",
".os",
".obj",
".bin",
".res",
".rc",
".S",
".cpp",
".manifest",
):
Utils.deleteFile(path, True)
else:
Utils.makePath(source_dir)
def cleanSourceDirectory(source_dir):
if Utils.isDir(source_dir):
for path, _filename in Utils.listDir(source_dir):
if Utils.getExtension(path) in (".cpp", ".hpp", ".c", ".o", ".os",
".obj", ".bin", ".res", ".rc",
".manifest"):
Utils.deleteFile(path, True)
else:
Utils.makePath(source_dir)
static_source_dir = Utils.joinpath(
source_dir,
"static"
)
if Utils.isDir(static_source_dir):
for path, _filename in Utils.listDir(static_source_dir):
if Utils.getExtension(path) in (".o", ".os", ".obj"):
Utils.deleteFile(path, True)
win32_source_dir = Utils.joinpath(
static_source_dir,
"win32_ucontext_src"
)
if Utils.getOS() == "Windows":
Utils.deleteFile(
Utils.joinpath(win32_source_dir, "fibers_win32.obj"),
False
)
def _detectBinaryPathDLLsWindows(original_dir, binary_filename, package_name):
result = set()
depends_exe = getDependsExePath()
# The search order by default prefers the system directory, where a
# wrong "PythonXX.dll" might be living.
with open(binary_filename + ".dwp", 'w') as dwp_file:
dwp_file.write("""\
KnownDLLs
SysPath
AppDir
{original_dir}
32BitSysDir
16BitSysDir
OSDir
AppPath
SxS
""".format(original_dir = "UserDir %s" % original_dir if original_dir is not None else "")
)
subprocess.call(
(
depends_exe,
"-c",
"-ot%s" % binary_filename + ".depends",
"-d:%s" % binary_filename + ".dwp",
"-f1",
"-pa1",
"-ps1",
binary_filename
),
env = _makeBinaryPathPathDLLSearchEnv(package_name),
)
inside = False
first = False
for line in open(binary_filename + ".depends"):
if "| Module Dependency Tree |" in line:
inside = True
first = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if ']' not in line:
continue
# Skip missing DLLs, apparently not needed anyway.
if '?' in line[:line.find(']')]:
continue
# Skip DLLs that failed to load, apparently not needed anyway.
if 'E' in line[:line.find(']')]:
continue
dll_filename = line[line.find(']')+2:-1]
# The executable itself is of course exempted. We cannot check its path
# because depends.exe mistreats unicode paths.
if first:
first = False
continue
assert Utils.isFile(dll_filename), dll_filename
dll_name = Utils.basename(dll_filename).upper()
# Win API can be assumed.
if dll_name.startswith("API-MS-WIN-") or \
dll_name.startswith("EXT-MS-WIN-"):
continue
if dll_name in ("SHELL32.DLL", "USER32.DLL", "KERNEL32.DLL",
"NTDLL.DLL", "NETUTILS.DLL", "LOGONCLI.DLL", "GDI32.DLL",
"RPCRT4.DLL", "ADVAPI32.DLL", "SSPICLI.DLL", "SECUR32.DLL",
"KERNELBASE.DLL", "WINBRAND.DLL", "DSROLE.DLL", "DNSAPI.DLL",
"SAMCLI.DLL", "WKSCLI.DLL", "SAMLIB.DLL", "WLDAP32.DLL",
"NTDSAPI.DLL", "CRYPTBASE.DLL", "W32TOPL", "WS2_32.DLL",
"SPPC.DLL", "MSSIGN32.DLL", "CERTCLI.DLL", "WEBSERVICES.DLL",
"AUTHZ.DLL", "CERTENROLL.DLL", "VAULTCLI.DLL", "REGAPI.DLL",
"BROWCLI.DLL", "WINNSI.DLL", "DHCPCSVC6.DLL", "PCWUM.DLL",
"CLBCATQ.DLL", "IMAGEHLP.DLL", "MSASN1.DLL", "DBGHELP.DLL",
"DEVOBJ.DLL", "DRVSTORE.DLL", "CABINET.DLL", "SCECLI.DLL",
"SPINF.DLL", "SPFILEQ.DLL", "GPAPI.DLL", "NETJOIN.DLL",
"W32TOPL.DLL", "NETBIOS.DLL", "DXGI.DLL", "DWRITE.DLL",
"D3D11.DLL", "WLANAPI.DLL", "WLANUTIL.DLL", "ONEX.DLL",
"EAPPPRXY.DLL", "MFPLAT.DLL", "AVRT.DLL", "ELSCORE.DLL",
"INETCOMM.DLL", "MSOERT2.DLL", "IEUI.DLL", "MSCTF.DLL",
"MSFEEDS.DLL", "UIAUTOMATIONCORE.DLL", "PSAPI.DLL",
"EFSADU.DLL", "MFC42U.DLL", "ODBC32.DLL", "OLEDLG.DLL",
"NETAPI32.DLL", "LINKINFO.DLL", "DUI70.DLL", "ADVPACK.DLL",
"NTSHRUI.DLL", "WINSPOOL.DRV", "EFSUTIL.DLL", "WINSCARD.DLL",
"SHDOCVW.DLL", "IEFRAME.DLL", "D2D1.DLL", "GDIPLUS.DLL",
"OCCACHE.DLL", "IEADVPACK.DLL", "MLANG.DLL", "MSI.DLL",
"MSHTML.DLL", "COMDLG32.DLL", "PRINTUI.DLL", "PUIAPI.DLL",
"ACLUI.DLL", "WTSAPI32.DLL", "FMS.DLL", "DFSCLI.DLL",
"HLINK.DLL", "MSRATING.DLL", "PRNTVPT.DLL", "IMGUTIL.DLL",
"MSLS31.DLL", "VERSION.DLL", "NORMALIZ.DLL", "IERTUTIL.DLL",
"WININET.DLL", "WINTRUST.DLL", "XMLLITE.DLL", "APPHELP.DLL",
"PROPSYS.DLL", "RSTRTMGR.DLL", "NCRYPT.DLL", "BCRYPT.DLL",
"MMDEVAPI.DLL", "MSILTCFG.DLL", "DEVMGR.DLL", "DEVRTL.DLL",
"NEWDEV.DLL", "VPNIKEAPI.DLL", "WINHTTP.DLL", "WEBIO.DLL",
"NSI.DLL", "DHCPCSVC.DLL", "CRYPTUI.DLL", "ESENT.DLL",
"DAVHLPR.DLL", "CSCAPI.DLL", "ATL.DLL", "OLEAUT32.DLL",
"SRVCLI.DLL", "RASDLG.DLL", "MPRAPI.DLL", "RTUTILS.DLL",
"RASMAN.DLL", "MPRMSG.DLL", "SLC.DLL", "CRYPTSP.DLL",
"RASAPI32.DLL", "TAPI32.DLL", "EAPPCFG.DLL", "NDFAPI.DLL",
"WDI.DLL", "COMCTL32.DLL", "UXTHEME.DLL", "IMM32.DLL",
"OLEACC.DLL", "WINMM.DLL", "WINDOWSCODECS.DLL", "DWMAPI.DLL",
"DUSER.DLL", "PROFAPI.DLL", "URLMON.DLL", "SHLWAPI.DLL",
"LPK.DLL", "USP10.DLL", "CFGMGR32.DLL", "MSIMG32.DLL",
"POWRPROF.DLL", "SETUPAPI.DLL", "WINSTA.DLL", "CRYPT32.DLL",
"IPHLPAPI.DLL", "MPR.DLL", "CREDUI.DLL", "NETPLWIZ.DLL",
"OLE32.DLL", "ACTIVEDS.DLL", "ADSLDPC.DLL", "USERENV.DLL",
"APPREPAPI.DLL", "BCP47LANGS.DLL", "BCRYPTPRIMITIVES.DLL",
"CERTCA.DLL", "CHARTV.DLL", "COMBASE.DLL", "COML2.DLL",
"DCOMP.DLL", "DPAPI.DLL", "DSPARSE.DLL", "FECLIENT.DLL",
"FIREWALLAPI.DLL", "FLTLIB.DLL", "MRMCORER.DLL", "NTASN1.DLL",
"SECHOST.DLL", "SETTINGSYNCPOLICY.DLL", "SHCORE.DLL", "TBS.DLL",
"TWINAPI.APPCORE.DLL", "TWINAPI.DLL", "VIRTDISK.DLL",
"WEBSOCKET.DLL", "WEVTAPI.DLL", "WINMMBASE.DLL", "WMICLNT.DLL"):
continue
result.add(
Utils.normcase(Utils.abspath(dll_filename))
)
Utils.deleteFile(binary_filename + ".depends", must_exist = True)
Utils.deleteFile(binary_filename + ".dwp", must_exist = True)
return result
def _detectBinaryPathDLLsWindows(binary_filename, package_name):
result = set()
depends_exe = getDependsExePath()
# The search order by default prefers the system directory, where a
# wrong "PythonXX.dll" might be living.
with open(binary_filename + ".dwp", 'w') as dwp_file:
dwp_file.write("""\
KnownDLLs
SysPath
AppDir
32BitSysDir
16BitSysDir
OSDir
AppPath
SxS
""")
subprocess.call(
(
depends_exe,
"-c",
"-ot%s" % binary_filename + ".depends",
"-d:%s" % binary_filename + ".dwp",
"-f1",
"-pa1",
"-ps1",
binary_filename
),
env = _makeBinaryPathPathDLLSearchEnv(package_name),
)
inside = False
for line in open(binary_filename + ".depends"):
if "| Module Dependency Tree |" in line:
inside = True
continue
if not inside:
continue
if "| Module List |" in line:
break
if ']' not in line:
continue
# Skip missing DLLs, apparently not needed anyway.
if '?' in line[:line.find(']')]:
continue
# Skip DLLs that failed to load, apparently not needed anyway.
if 'E' in line[:line.find(']')]:
continue
dll_filename = line[line.find(']')+2:-1]
assert Utils.isFile(dll_filename), dll_filename
# The executable itself is of course exempted.
if Utils.normcase(dll_filename) == \
Utils.normcase(Utils.abspath(binary_filename)):
continue
dll_name = Utils.basename(dll_filename).upper()
# Win API can be assumed.
if dll_name.startswith("API-MS-WIN-") or \
dll_name.startswith("EXT-MS-WIN-"):
continue
if dll_name in ("SHELL32.DLL", "USER32.DLL", "KERNEL32.DLL",
"NTDLL.DLL", "NETUTILS.DLL", "LOGONCLI.DLL", "GDI32.DLL",
"RPCRT4.DLL", "ADVAPI32.DLL", "SSPICLI.DLL", "SECUR32.DLL",
"KERNELBASE.DLL", "WINBRAND.DLL", "DSROLE.DLL", "DNSAPI.DLL",
"SAMCLI.DLL", "WKSCLI.DLL", "SAMLIB.DLL", "WLDAP32.DLL",
"NTDSAPI.DLL", "CRYPTBASE.DLL", "W32TOPL", "WS2_32.DLL",
"SPPC.DLL", "MSSIGN32.DLL", "CERTCLI.DLL", "WEBSERVICES.DLL",
"AUTHZ.DLL", "CERTENROLL.DLL", "VAULTCLI.DLL", "REGAPI.DLL",
"BROWCLI.DLL", "WINNSI.DLL", "DHCPCSVC6.DLL", "PCWUM.DLL",
"CLBCATQ.DLL", "IMAGEHLP.DLL", "MSASN1.DLL", "DBGHELP.DLL",
"DEVOBJ.DLL", "DRVSTORE.DLL", "CABINET.DLL", "SCECLI.DLL",
"SPINF.DLL", "SPFILEQ.DLL", "GPAPI.DLL", "NETJOIN.DLL",
"W32TOPL.DLL", "NETBIOS.DLL", "DXGI.DLL", "DWRITE.DLL",
"D3D11.DLL", "WLANAPI.DLL", "WLANUTIL.DLL", "ONEX.DLL",
"EAPPPRXY.DLL", "MFPLAT.DLL", "AVRT.DLL", "ELSCORE.DLL",
"INETCOMM.DLL", "MSOERT2.DLL", "IEUI.DLL", "MSCTF.DLL",
"MSFEEDS.DLL", "UIAUTOMATIONCORE.DLL", "PSAPI.DLL",
"EFSADU.DLL", "MFC42U.DLL", "ODBC32.DLL", "OLEDLG.DLL",
"NETAPI32.DLL", "LINKINFO.DLL", "DUI70.DLL", "ADVPACK.DLL",
"NTSHRUI.DLL", "WINSPOOL.DRV", "EFSUTIL.DLL", "WINSCARD.DLL",
"SHDOCVW.DLL", "IEFRAME.DLL", "D2D1.DLL", "GDIPLUS.DLL",
"OCCACHE.DLL", "IEADVPACK.DLL", "MLANG.DLL", "MSI.DLL",
"MSHTML.DLL", "COMDLG32.DLL", "PRINTUI.DLL", "PUIAPI.DLL",
"ACLUI.DLL", "WTSAPI32.DLL", "FMS.DLL", "DFSCLI.DLL",
"HLINK.DLL", "MSRATING.DLL", "PRNTVPT.DLL", "IMGUTIL.DLL",
"MSLS31.DLL", "VERSION.DLL", "NORMALIZ.DLL", "IERTUTIL.DLL",
"WININET.DLL", "WINTRUST.DLL", "XMLLITE.DLL", "APPHELP.DLL",
"PROPSYS.DLL", "RSTRTMGR.DLL", "NCRYPT.DLL", "BCRYPT.DLL",
"MMDEVAPI.DLL", "MSILTCFG.DLL", "DEVMGR.DLL", "DEVRTL.DLL",
"NEWDEV.DLL", "VPNIKEAPI.DLL", "WINHTTP.DLL", "WEBIO.DLL",
"NSI.DLL", "DHCPCSVC.DLL", "CRYPTUI.DLL", "ESENT.DLL",
"DAVHLPR.DLL", "CSCAPI.DLL", "ATL.DLL", "OLEAUT32.DLL",
"SRVCLI.DLL", "RASDLG.DLL", "MPRAPI.DLL", "RTUTILS.DLL",
"RASMAN.DLL", "MPRMSG.DLL", "SLC.DLL", "CRYPTSP.DLL",
"RASAPI32.DLL", "TAPI32.DLL", "EAPPCFG.DLL", "NDFAPI.DLL",
"WDI.DLL", "COMCTL32.DLL", "UXTHEME.DLL", "IMM32.DLL",
"OLEACC.DLL", "WINMM.DLL", "WINDOWSCODECS.DLL", "DWMAPI.DLL",
"DUSER.DLL", "PROFAPI.DLL", "URLMON.DLL", "SHLWAPI.DLL",
"LPK.DLL", "USP10.DLL", "CFGMGR32.DLL", "MSIMG32.DLL",
"POWRPROF.DLL", "SETUPAPI.DLL", "WINSTA.DLL", "CRYPT32.DLL",
"IPHLPAPI.DLL", "MPR.DLL", "CREDUI.DLL", "NETPLWIZ.DLL",
"OLE32.DLL", "ACTIVEDS.DLL", "ADSLDPC.DLL", "USERENV.DLL",
"APPREPAPI.DLL", "BCP47LANGS.DLL", "BCRYPTPRIMITIVES.DLL",
"CERTCA.DLL", "CHARTV.DLL", "COMBASE.DLL", "DCOMP.DLL",
"DPAPI.DLL", "DSPARSE.DLL", "FECLIENT.DLL", "FIREWALLAPI.DLL",
"FLTLIB.DLL", "MRMCORER.DLL", "MSVCRT.DLL",
"NINPUT.DLL", "NTASN1.DLL", "PCACLI.DLL", "RTWORKQ.DLL",
"SECHOST.DLL", "SETTINGSYNCPOLICY.DLL", "SHCORE.DLL",
"TBS.DLL", "TWINAPI.DLL", "TWINAPI.APPCORE.DLL", "VIRTDISK.DLL",
"WEBSOCKET.DLL", "WEVTAPI.DLL", "WINMMBASE.DLL", "WMICLNT.DLL"):
continue
result.add(
Utils.normcase(Utils.abspath(dll_filename))
)
Utils.deleteFile(binary_filename + ".depends", must_exist = True)
Utils.deleteFile(binary_filename + ".dwp", must_exist = True)
return result
def getDependsExePath():
""" Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if Utils.getArchitecture() == "x86":
depends_url = "http://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "http://dependencywalker.com/depends22_x64.zip"
if "APPDATA" not in os.environ:
sys.exit("Error, standalone mode cannot find 'APPDATA' environment.")
nuitka_app_dir = Utils.joinpath(os.environ["APPDATA"], "nuitka")
if not Utils.isDir(nuitka_app_dir):
Utils.makePath(nuitka_app_dir)
nuitka_depends_zip = Utils.joinpath(nuitka_app_dir,
Utils.basename(depends_url))
if not Utils.isFile(nuitka_depends_zip):
Tracing.printLine("""\
Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool
to analyze the dependencies of Python extension modules. Is it OK to download
and put it in APPDATA (no installer needed, cached, one time question).""")
reply = raw_input("Proceed and download? [Yes]/No ")
if reply.lower() in ("no", 'n'):
sys.exit(
"Nuitka does not work in --standalone on Windows without.")
info("Downloading '%s'" % depends_url)
urlretrieve(depends_url, nuitka_depends_zip)
nuitka_depends_dir = Utils.joinpath(nuitka_app_dir,
Utils.getArchitecture())
if not Utils.isDir(nuitka_depends_dir):
os.makedirs(nuitka_depends_dir)
depends_exe = os.path.join(nuitka_depends_dir, "depends.exe")
if not Utils.isFile(depends_exe):
info("Extracting to '%s'" % depends_exe)
import zipfile
try:
depends_zip = zipfile.ZipFile(nuitka_depends_zip)
depends_zip.extractall(nuitka_depends_dir)
except Exception: # Catching anything zip throws, pylint:disable=W0703
info("Problem with the downloaded zip file, deleting it.")
Utils.deleteFile(depends_exe, must_exist=False)
Utils.deleteFile(nuitka_depends_zip, must_exist=True)
sys.exit("Error, need '%s' as extracted from '%s'." %
(depends_exe, depends_url))
assert Utils.isFile(depends_exe)
return depends_exe
def getDependsExePath():
""" Return the path of depends.exe (for Windows).
Will prompt the user to download if not already cached in AppData
directory for Nuitka.
"""
if Utils.getArchitecture() == "x86":
depends_url = "http://dependencywalker.com/depends22_x86.zip"
else:
depends_url = "http://dependencywalker.com/depends22_x64.zip"
if "APPDATA" not in os.environ:
sys.exit("Error, standalone mode cannot find 'APPDATA' environment.")
nuitka_app_dir = Utils.joinpath(os.environ["APPDATA"], "nuitka")
if not Utils.isDir(nuitka_app_dir):
Utils.makePath(nuitka_app_dir)
nuitka_depends_zip = Utils.joinpath(
nuitka_app_dir,
Utils.basename(depends_url)
)
if not Utils.isFile(nuitka_depends_zip):
Tracing.printLine("""\
Nuitka will make use of Dependency Walker (http://dependencywalker.com) tool
to analyze the dependencies of Python extension modules. Is it OK to download
and put it in APPDATA (no installer needed, cached, one time question).""")
reply = raw_input("Proceed and download? [Yes]/No ")
if reply.lower() in ("no", 'n'):
sys.exit("Nuitka does not work in --standalone on Windows without.")
info("Downloading '%s'" % depends_url)
urlretrieve(
depends_url,
nuitka_depends_zip
)
nuitka_depends_dir = Utils.joinpath(
nuitka_app_dir,
Utils.getArchitecture()
)
if not Utils.isDir(nuitka_depends_dir):
os.makedirs(nuitka_depends_dir)
depends_exe = os.path.join(
nuitka_depends_dir,
"depends.exe"
)
if not Utils.isFile(depends_exe):
info("Extracting to '%s'" % depends_exe)
import zipfile
try:
depends_zip = zipfile.ZipFile(nuitka_depends_zip)
depends_zip.extractall(nuitka_depends_dir)
except Exception: # Catching anything zip throws, pylint:disable=W0703
info("Problem with the downloaded zip file, deleting it.")
Utils.deleteFile(depends_exe, must_exist = False)
Utils.deleteFile(nuitka_depends_zip, must_exist = True)
sys.exit(
"Error, need '%s' as extracted from '%s'." % (
depends_exe,
depends_url
)
)
assert Utils.isFile(depends_exe)
return depends_exe