def _checkPluginPath(plugin_filename, module_package):
plugin_info = considerFilename(module_package=module_package,
module_filename=plugin_filename)
if plugin_info is not None:
module, added = recurseTo(module_filename=plugin_info[0],
module_relpath=plugin_info[1],
module_package=module_package)
if module:
if not added:
warning("Recursed to %s '%s' at '%s' twice.",
"package" if module.isPythonPackage() else "module",
module.getName(), plugin_info[0])
if module.isPythonPackage():
package_dir = Utils.dirname(module.getFilename())
for sub_path, sub_filename in Utils.listDir(package_dir):
if sub_filename == "__init__.py":
continue
assert sub_path != plugin_filename, package_dir
if Importing.isPackageDir(sub_path) or sub_path.endswith(
".py"):
_checkPluginPath(sub_path, module.getFullName())
else:
warning("Failed to include module from '%s'.", plugin_info[0])
def runScons( options, quiet ):
# For the scons file to find the static C++ files and include path. The scons file is
# unable to use __file__ for the task.
os.environ[ "NUITKA_SCONS" ] = getSconsDataPath()
if os.name == "nt":
# On Windows this Scons variable must be set by us.
os.environ[ "SCONS_LIB_DIR" ] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.0.1" )
# Also, for MinGW we can avoid the user having to add the path if he used the
# default path or installed it on the same drive by appending to the PATH variable
# before executing scons.
os.environ[ "PATH" ] += r";\MinGW\bin;C:\MinGW\bin"
scons_command = """%(python)s %(binary)s %(quiet)s -f %(scons_file)s --jobs %(job_limit)d %(options)s""" % {
"python" : sys.executable if Utils.python_version < 300 else "python",
"binary" : getSconsBinaryPath(),
"quiet" : "--quiet" if quiet else "",
"scons_file" : Utils.joinpath( getSconsDataPath(), "SingleExe.scons" ),
"job_limit" : Options.getJobLimit(),
"options" : " ".join( "%s=%s" % ( key, value ) for key, value in options.items() )
}
if Options.isShowScons():
Tracing.printLine( "Scons command:", scons_command )
return 0 == os.system( scons_command )
def checkPluginFilenamePattern(pattern):
import sys, glob
debug(
"Checking plug-in pattern '%s':",
pattern,
)
if Utils.isDir(pattern):
sys.exit("Error, pattern cannot be a directory name.")
found = False
for filename in glob.iglob(pattern):
if filename.endswith(".pyc"):
continue
if not Utils.isFile(filename):
continue
found = True
_checkPluginPath(filename, None)
if not found:
warning("Didn't match any files against pattern '%s'." % pattern)
def considerImplicitImports(self, signal_change):
for module_name, module_package in self.getImplicitImports():
_module_package, _module_name, module_filename = \
Importing.findModule(
source_ref = self.source_ref,
module_name = module_name,
parent_package = module_package,
level = -1,
warn = True
)
if module_filename is None:
sys.exit(
"Error, implicit module '%s' expected by '%s' not found" % (
module_name,
self.getFullName()
)
)
elif Utils.isDir(module_filename):
module_kind = "py"
elif module_filename.endswith(".py"):
module_kind = "py"
elif module_filename.endswith(".so"):
module_kind = "shlib"
elif module_filename.endswith(".pyd"):
module_kind = "shlib"
else:
assert False, module_filename
from nuitka.tree import Recursion
decision, reason = Recursion.decideRecursion(
module_filename = module_filename,
module_name = module_name,
module_package = module_package,
module_kind = module_kind
)
assert decision or reason == "Module is frozen."
if decision:
module_relpath = Utils.relpath(module_filename)
imported_module, added_flag = Recursion.recurseTo(
module_package = module_package,
module_filename = module_filename,
module_relpath = module_relpath,
module_kind = module_kind,
reason = reason
)
from nuitka.ModuleRegistry import addUsedModule
addUsedModule(imported_module)
if added_flag:
signal_change(
"new_code",
imported_module.getSourceReference(),
"Recursed to module."
)
def _detectedSourceFile(filename, module_name, result, is_late):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(filename=filename,
module_name="_collections_abc",
result=result,
is_late=is_late)
source_code = readSourceCodeFromFilename(filename)
if Utils.python_version >= 300:
filename = filename.decode("utf-8")
if module_name == "site":
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep, Utils.basename(filename), source_code)
debug("Freezing module '%s' (from '%s').", module_name, filename)
result.append(
(module_name, marshal.dumps(compile(source_code, filename, "exec")),
Utils.basename(filename) == "__init__.py", filename, is_late))
module_names.add(module_name)
def checkPluginPath(plugin_filename, module_package):
debug(
"Checking top level plugin path %s %s",
plugin_filename,
module_package
)
plugin_info = considerFilename(
module_package = module_package,
module_filename = plugin_filename
)
if plugin_info is not None:
# File or package makes a difference, handle that
if Utils.isFile(plugin_info[0]) or \
Importing.isPackageDir(plugin_info[0]):
_checkPluginPath(plugin_filename, module_package)
elif Utils.isDir(plugin_info[0]):
for sub_path, sub_filename in Utils.listDir(plugin_info[0]):
assert sub_filename != "__init__.py"
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, None)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
else:
warning("Failed to recurse to directory '%s'.", plugin_filename)
def checkPluginPath(plugin_filename, module_package):
debug(
"Checking top level plugin path %s %s",
plugin_filename,
module_package
)
plugin_info = considerFilename(
module_package = module_package,
module_filename = plugin_filename
)
if plugin_info is not None:
# File or package makes a difference, handle that
if Utils.isFile(plugin_info[0]) or \
Importing.isPackageDir(plugin_info[0]):
_checkPluginPath(plugin_filename, module_package)
elif Utils.isDir(plugin_info[0]):
for sub_path, sub_filename in Utils.listDir(plugin_info[0]):
assert sub_filename != "__init__.py"
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, None)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
else:
warning("Failed to recurse to directory '%s'.", plugin_filename)
def detectBinaryDLLs(binary_filename, package_name):
""" Detect the DLLs used by a binary.
Using ldd (Linux), depends.exe (Windows), or otool (MacOS) the list
of used DLLs is retrieved.
"""
if Utils.getOS() in ("Linux", "NetBSD"):
# TODO: FreeBSD may work the same way, not tested.
return _detectBinaryPathDLLsLinuxBSD(
binary_filename = binary_filename
)
elif Utils.getOS() == "Windows":
return _detectBinaryPathDLLsWindows(
binary_filename = binary_filename,
package_name = package_name
)
elif Utils.getOS() == "Darwin":
return _detectBinaryPathDLLsLinuxBSD(
binary_filename = binary_filename
)
else:
# Support your platform above.
assert False, Utils.getOS()
def considerImplicitImports(self, signal_change):
for module_name, module_package in self.getImplicitImports():
_module_package, _module_name, module_filename = \
Importing.findModule(
source_ref = self.source_ref,
module_name = module_name,
parent_package = module_package,
level = -1,
warn = True
)
if module_filename is None:
sys.exit(
"Error, implicit module '%s' expected by '%s' not found" % (
module_name,
self.getFullName()
)
)
elif Utils.isDir(module_filename):
module_kind = "py"
elif module_filename.endswith(".py"):
module_kind = "py"
elif module_filename.endswith(".so"):
module_kind = "shlib"
elif module_filename.endswith(".pyd"):
module_kind = "shlib"
else:
assert False, module_filename
from nuitka.tree import Recursion
decision, reason = Recursion.decideRecursion(
module_filename = module_filename,
module_name = module_name,
module_package = module_package,
module_kind = module_kind
)
assert decision or reason == "Module is frozen."
if decision:
module_relpath = Utils.relpath(module_filename)
imported_module, added_flag = Recursion.recurseTo(
module_package = module_package,
module_filename = module_filename,
module_relpath = module_relpath,
module_kind = module_kind,
reason = reason
)
from nuitka.ModuleRegistry import addUsedModule
addUsedModule(imported_module)
if added_flag:
signal_change(
"new_code",
imported_module.getSourceReference(),
"Recursed to module."
)
def getSconsBinaryCall():
""" Return a way to execute Scons.
Using potentially inline copy if no system Scons is available
or if we are on Windows.
"""
if Utils.isFile("/usr/bin/scons"):
return ["/usr/bin/scons"]
else:
return [getPython2ExePath(), Utils.joinpath(getSconsInlinePath(), "bin", "scons.py")]
def getVariableCodeName(in_context, variable):
if in_context:
# Closure case:
return "closure_" + Utils.encodeNonAscii(variable.getName())
elif variable.isParameterVariable():
return "par_" + Utils.encodeNonAscii(variable.getName())
elif variable.isTempVariable():
return "tmp_" + Utils.encodeNonAscii(variable.getName())
else:
return "var_" + Utils.encodeNonAscii(variable.getName())
def getSconsBinaryCall():
""" Return a way to execute Scons.
Using potentially inline copy if no system Scons is available
or if we are on Windows.
"""
if Utils.isFile("/usr/bin/scons"):
return ["/usr/bin/scons"]
else:
return [
getPython2ExePath(),
Utils.joinpath(getSconsInlinePath(), "bin", "scons.py")
]
def __init__(self, *args):
QtGui.QDialog.__init__(self, *args)
ui_dir = Utils.dirname(__file__)
ui_filename = Utils.joinpath(ui_dir, "dialogs", "InspectPythonTree.ui")
uic.loadUi(ui_filename, self)
self.treeview_nodes.setSelectionMode(self.treeview_nodes.SingleSelection)
self.displayed = None
self.source_code = None
self.model = None
self.moving = None
def copyUsedDLLs(dist_dir, binary_filename, standalone_entry_points):
dll_map = []
used_dlls = detectUsedDLLs(standalone_entry_points)
for dll_filename1, sources1 in iterItems(used_dlls):
for dll_filename2, sources2 in iterItems(used_dlls):
if dll_filename1 == dll_filename2:
continue
# Colliding basenames are an issue to us.
if Utils.basename(dll_filename1) != Utils.basename(dll_filename2):
continue
dll_name = Utils.basename(dll_filename1)
if Options.isShowInclusion():
info("""Colliding DLL names for %s, checking identity of \
'%s' <-> '%s'.""" % (
dll_name,
dll_filename1,
dll_filename2,
))
# Check that if a DLL has the same name, if it's identical,
# happens at least for OSC and Fedora 20.
import filecmp
if filecmp.cmp(dll_filename1, dll_filename2):
continue
sys.exit("""Error, conflicting DLLs for '%s' \
(%s used by %s different from %s used by %s).""" %
(dll_name, dll_filename1, ", ".join(sources1),
dll_filename2, ", ".join(sources2)))
for dll_filename, sources in iterItems(used_dlls):
dll_name = Utils.basename(dll_filename)
target_path = Utils.joinpath(dist_dir, dll_name)
shutil.copy(dll_filename, target_path)
dll_map.append((dll_filename, dll_name))
if Options.isShowInclusion():
info("Included used shared library '%s' (used by %s)." %
(dll_filename, ", ".join(sources)))
if Utils.getOS() == "Darwin":
# For MacOS, the binary needs to be changed to reflect the DLL
# location in the dist folder.
fixupBinaryDLLPaths(binary_filename, dll_map)
if Utils.getOS() == "Linux":
# For Linux, the rpath of libraries may be an issue.
for _original_path, dll_filename in dll_map:
removeSharedLibraryRPATH(Utils.joinpath(dist_dir, dll_filename))
for standalone_entry_point in standalone_entry_points[1:]:
removeSharedLibraryRPATH(standalone_entry_point[0])
def __init__(self, *args):
QtGui.QDialog.__init__( self, *args )
ui_dir = Utils.dirname( __file__ )
ui_filename = Utils.joinpath( ui_dir, "dialogs", "InspectPythonTree.ui" )
uic.loadUi( ui_filename, self )
self.treeview_nodes.setSelectionMode( self.treeview_nodes.SingleSelection )
self.displayed = None
self.source_code = None
self.model = None
self.moving = None
def runScons(options, quiet):
# For the scons file to find the static C++ files and include path. The
# scons file is unable to use __file__ for the task.
os.environ["NUITKA_SCONS"] = getSconsDataPath()
if Utils.getOS() == "Windows":
# On Windows this Scons variable must be set by us.
os.environ["SCONS_LIB_DIR"] = Utils.joinpath(
getSconsInlinePath(),
"lib",
"scons-2.3.0"
)
# Also, for MinGW we can avoid the user having to add the path if he
# used the default path or installed it on the same drive by appending
# to the PATH variable before executing scons.
os.environ["PATH"] += r";\MinGW\bin;C:\MinGW\bin"
scons_command = getSconsBinaryCall()
if quiet:
scons_command.append("--quiet")
scons_command += [
# The scons file
"-f",
Utils.joinpath(getSconsDataPath(), "SingleExe.scons"),
# Parallel compilation.
"--jobs",
str(Options.getJobLimit()),
# Do not warn about deprecations of Scons
"--warn=no-deprecated",
# Don't load "site_scons" at all.
"--no-site-dir",
]
if Options.isShowScons():
scons_command.append("--debug=explain")
# Option values to provide to scons.
for key, value in options.items():
scons_command += [key + "=" + value]
if Options.isShowScons():
Tracing.printLine("Scons command:", " ".join(scons_command))
return 0 == subprocess.call(scons_command)
def runScons(options, quiet):
# For the scons file to find the static C++ files and include path. The scons file is
# unable to use __file__ for the task.
os.environ["NUITKA_SCONS"] = getSconsDataPath()
if os.name == "nt":
# On Windows this Scons variable must be set by us.
os.environ["SCONS_LIB_DIR"] = Utils.joinpath(getSconsInlinePath(),
"lib", "scons-2.0.1")
# Also, for MinGW we can avoid the user having to add the path if he used the
# default path or installed it on the same drive by appending to the PATH variable
# before executing scons.
os.environ["PATH"] += r";\MinGW\bin;C:\MinGW\bin"
# Scons is Python2 only, so we need to make the system find a suitable Python binary.
if Utils.python_version < 300:
python_exe = sys.executable
elif os.name == "nt":
if os.path.exists(r"c:\Python27\python.exe"):
python_exe = r"c:\Python27\python.exe"
elif os.path.exists(r"c:\Python26\python.exe"):
python_exe = r"c:\Python26\python.exe"
else:
sys.exit(
"""Error, need to find Python2 executable under C:\\Python26 or \
C:\\Python27 to execute scons which is not Python3 compatible.""")
else:
python_exe = "python"
scons_command = """%(python)s %(binary)s %(quiet)s --warn=no-deprecated -f %(scons_file)s --jobs %(job_limit)d %(options)s""" % {
"python":
python_exe,
"binary":
getSconsBinaryPath(),
"quiet":
"--quiet" if quiet else "",
"scons_file":
Utils.joinpath(getSconsDataPath(), "SingleExe.scons"),
"job_limit":
Options.getJobLimit(),
"options":
" ".join("%s=%s" % (key, value) for key, value in options.items())
}
if Options.isShowScons():
Tracing.printLine("Scons command:", scons_command)
return 0 == os.system(scons_command)
def buildModuleTree(filename, package, is_top, is_main):
module, source_ref, source_filename = decideModuleTree(
filename = filename,
package = package,
is_top = is_top,
is_main = is_main,
is_shlib = False
)
addImportedModule(
module_relpath = Utils.relpath(filename),
imported_module = module
)
# If there is source code associated (not the case for namespace packages of
# Python3.3 or higher, then read it.
if source_filename is not None:
createModuleTree(
module = module,
source_ref = source_ref,
source_filename = source_filename,
is_main = is_main
)
return module
def buildModuleTree(filename, package, is_top, is_main):
module, source_ref, source_filename = decideModuleTree(
filename = filename,
package = package,
is_top = is_top,
is_main = is_main,
is_shlib = False
)
addImportedModule(
module_relpath = Utils.relpath(filename),
imported_module = module
)
# If there is source code associated (not the case for namespace packages of
# Python3.3 or higher, then read it.
if source_filename is not None:
createModuleTree(
module = module,
source_ref = source_ref,
source_filename = source_filename,
is_main = is_main
)
return module
def getRunTimeFilename(self):
if Options.isStandaloneMode():
filename = self.getCompileTimeFilename()
full_name = self.getFullName()
result = Utils.basename(filename)
current = filename
for _i in range(full_name.count('.')):
current = Utils.dirname(current)
result = Utils.joinpath(Utils.basename(current), result)
return result
else:
return self.getCompileTimeFilename()
def createModuleTree(module, source_ref, source_filename, is_main):
if Options.isShowProgress():
memory_watch = Utils.MemoryWatch()
source_code = readSourceCodeFromFilename(source_filename)
module_body = buildParseTree(
provider = module,
source_code = source_code,
source_ref = source_ref,
is_module = True,
is_main = is_main
)
module.setBody(
module_body
)
completeVariableClosures(module)
if Options.isShowProgress():
memory_watch.finish()
Tracing.printLine(
"Memory usage changed loading module '%s': %s" % (
module.getFullName(),
memory_watch.asStr()
)
)
def detectEarlyImports():
if Options.freezeAllStdlib():
stdlib_modules = set()
# Scan the standard library paths (multiple in case of virtualenv.
for stdlib_dir in getStandardLibraryPaths():
for module_name in scanStandardLibraryPath(stdlib_dir):
stdlib_modules.add(module_name)
import_code = 'imports = ' + repr(sorted(stdlib_modules)) + '\n'\
'for imp in imports:\n' \
' try:\n' \
' __import__(imp)\n' \
' except ImportError:\n' \
' pass\n'
else:
# TODO: Should recursively include all of encodings module.
import_code = "import encodings.utf_8;import encodings.ascii;import encodings.idna;"
if Utils.getOS() == "Windows":
import_code += "import encodings.mbcs;import encodings.cp437;"
# String method hex depends on it.
if Utils.python_version < 300:
import_code += "import encodings.hex_codec;"
import_code += "import locale;"
result = _detectImports(import_code, False)
debug("Finished detecting early imports.")
return result
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name=module.getFullName()))
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
_optimizeModulePass(module=module, tag_set=tag_set)
if not tag_set:
break
touched = True
if _progress:
memory_watch.finish()
printLine("Memory usage changed during optimization of '%s': %s" %
(module.getFullName(), memory_watch.asStr()))
return touched
def removeSharedLibraryRPATH(filename):
process = subprocess.Popen(["readelf", "-d", filename],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
stdout, _stderr = process.communicate()
retcode = process.poll()
assert retcode == 0, filename
for line in stdout.split(b"\n"):
if b"RPATH" in line:
if Options.isShowInclusion():
info("Removing 'RPATH' setting from '%s'.", filename)
if not Utils.isExecutableCommand("chrpath"):
sys.exit("""\
Error, needs 'chrpath' on your system, due to 'RPATH' settings in used shared
libraries that need to be removed.""")
process = subprocess.Popen(["chrpath", "-d", filename],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=False)
process.communicate()
retcode = process.poll()
assert retcode == 0, filename
def optimize():
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine(
"""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name = current_module.getFullName(),
remaining = ModuleRegistry.remainingCount(),
memory = Utils.getHumanReadableProcessMemoryUsage()
)
)
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
if finished:
break
def removeSharedLibraryRPATH(filename):
process = subprocess.Popen(
["readelf", "-d", filename],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
shell = False
)
stdout, _stderr = process.communicate()
retcode = process.poll()
assert retcode == 0, filename
for line in stdout.split(b"\n"):
if b"RPATH" in line:
if Options.isShowInclusion():
info("Removing 'RPATH' setting from '%s'.", filename)
if not Utils.isExecutableCommand("chrpath"):
sys.exit(
"""\
Error, needs 'chrpath' on your system, due to 'RPATH' settings in used shared
libraries that need to be removed."""
)
process = subprocess.Popen(
["chrpath", "-d", filename],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
shell = False
)
process.communicate()
retcode = process.poll()
assert retcode == 0, filename
def optimize():
while True:
finished = True
ModuleRegistry.startTraversal()
while True:
current_module = ModuleRegistry.nextModule()
if current_module is None:
break
if _progress:
printLine("""\
Optimizing module '{module_name}', {remaining:d} more modules to go \
after that. Memory usage {memory}:""".format(
module_name=current_module.getFullName(),
remaining=ModuleRegistry.remainingCount(),
memory=Utils.getHumanReadableProcessMemoryUsage()))
if current_module.isPythonShlibModule():
optimizeShlibModule(current_module)
else:
changed = optimizePythonModule(current_module)
if changed:
finished = False
for current_module in ModuleRegistry.getDoneModules():
if not current_module.isPythonShlibModule():
optimizeVariables(current_module)
if finished:
break
def attemptRecursion( self, module ):
if not Options.shallMakeModule():
# Make sure the package is recursed to.
module_package = module.getPackage()
if module_package is not None:
package_package, _package_module_name, package_filename = Importing.findModule(
source_ref = module.getSourceReference(),
module_name = module_package,
parent_package = None,
level = 1
)
imported_module, added_flag = Recursion.recurseTo(
module_package = package_package,
module_filename = package_filename,
module_relpath = Utils.relpath( package_filename )
)
if added_flag:
self.signalChange(
"new_code",
imported_module.getSourceReference(),
"Recursed to module package."
)
def optimizePythonModule(module):
if _progress:
printLine(
"Doing module local optimizations for '{module_name}'.".format(
module_name=module.getFullName()))
# The tag set is global, so it can react to changes without context.
# pylint: disable=W0603
global tag_set
tag_set = TagSet()
touched = False
if _progress:
memory_watch = Utils.MemoryWatch()
while True:
tag_set.clear()
_optimizeModulePass(module=module)
if not tag_set:
break
touched = True
if _progress:
memory_watch.finish()
printLine("Memory usage changed during optimization of '%s': %s" %
(module.getFullName(), memory_watch.asStr()))
return touched
def _getPython2ExePathWindows():
# Shortcuts for the default installation directories, to avoid going to
# registry at all unless necessary. Any Python2 will do for Scons, so it
# can be avoided.
if os.path.isfile(r"c:\Python27\python.exe"):
return r"c:\Python27\python.exe"
elif os.path.isfile(r"c:\Python26\python.exe"):
return r"c:\Python26\python.exe"
# Windows only code, pylint: disable=E0602,F0401,I0021
try:
import _winreg as winreg
except ImportError:
import winreg # lint:ok
for search in ("2.7", "2.6"):
for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
try:
key = winreg.OpenKey(
winreg.HKEY_LOCAL_MACHINE,
r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search, 0,
winreg.KEY_READ | arch_key)
return Utils.joinpath(winreg.QueryValue(key, ""), "python.exe")
except WindowsError: # lint:ok
pass
def _getPython2ExePathWindows():
# Shortcuts for the default installation directories, to avoid going to
# registry at all.
if os.path.isfile(r"c:\Python27\python.exe"):
return r"c:\Python27\python.exe"
elif os.path.isfile(r"c:\Python26\python.exe"):
return r"c:\Python26\python.exe"
# Windows only code, pylint: disable=E0602,F0401
try:
import _winreg as winreg
except ImportError:
import winreg # lint:ok
for search in ("2.7", "2.6"):
for arch_key in 0, winreg.KEY_WOW64_32KEY, winreg.KEY_WOW64_64KEY:
try:
key = winreg.OpenKey(
winreg.HKEY_LOCAL_MACHINE,
r"SOFTWARE\Python\PythonCore\%s\InstallPath" % search,
0,
winreg.KEY_READ | arch_key,
)
return Utils.joinpath(winreg.QueryValue(key, ""), "python.exe")
except WindowsError: # lint:ok
pass
def detectEarlyImports():
if Options.freezeAllStdlib():
stdlib_modules = set()
# Scan the standard library paths (multiple in case of virtualenv.
for stdlib_dir in getStandardLibraryPaths():
for module_name in scanStandardLibraryPath(stdlib_dir):
stdlib_modules.add(module_name)
import_code = 'imports = ' + repr(sorted(stdlib_modules)) + '\n'\
'for imp in imports:\n' \
' try:\n' \
' __import__(imp)\n' \
' except ImportError:\n' \
' pass\n'
else:
# TODO: Should recursively include all of encodings module.
import_code = "import encodings.utf_8;import encodings.ascii;import encodings.idna;"
if Utils.getOS() == "Windows":
import_code += "import encodings.mbcs;import encodings.cp437;"
# String method hex depends on it.
if Utils.python_version < 300:
import_code += "import encodings.hex_codec;"
import_code += "import locale;"
result = _detectImports(import_code, False)
debug("Finished detecting early imports.")
return result
def runScons(options, quiet):
# For the scons file to find the static C++ files and include path. The
# scons file is unable to use __file__ for the task.
os.environ["NUITKA_SCONS"] = getSconsDataPath()
if Utils.getOS() == "Windows":
# On Windows this Scons variable must be set by us.
os.environ["SCONS_LIB_DIR"] = Utils.joinpath(
getSconsInlinePath(),
"lib",
"scons-2.3.2"
)
scons_command = getSconsBinaryCall()
if quiet:
scons_command.append("--quiet")
scons_command += [
# The scons file
"-f",
Utils.joinpath(getSconsDataPath(), "SingleExe.scons"),
# Parallel compilation.
"--jobs",
str(Options.getJobLimit()),
# Do not warn about deprecations of Scons
"--warn=no-deprecated",
# Don't load "site_scons" at all.
"--no-site-dir",
]
if Options.isShowScons():
scons_command.append("--debug=explain")
# Option values to provide to scons.
for key, value in options.items():
scons_command += [key + "=" + value]
if Options.isShowScons():
Tracing.printLine("Scons command:", " ".join(scons_command))
return 0 == subprocess.call(scons_command, shell = False)
def _consider(self, constraint_collection, module_filename,
module_package):
assert module_package is None or (type(module_package) is str
and module_package != "")
module_filename = Utils.normpath(module_filename)
if Utils.isDir(module_filename):
module_name = Utils.basename(module_filename)
elif module_filename.endswith(".py"):
module_name = Utils.basename(module_filename)[:-3]
else:
module_name = None
if module_name is not None:
decision = self._decide(module_filename, module_name,
module_package)
if decision:
module_relpath = Utils.relpath(module_filename)
return self._recurseTo(
constraint_collection=constraint_collection,
module_package=module_package,
module_filename=module_filename,
module_relpath=module_relpath)
elif decision is None:
if module_package is None:
module_fullpath = module_name
else:
module_fullpath = module_package + "." + module_name
if module_filename not in self._warned_about:
self._warned_about.add(module_filename)
warning( # long message, but shall be like it, pylint: disable=C0301
"""\
Not recursing to '%(full_path)s' (%(filename)s), please specify \
--recurse-none (do not warn), \
--recurse-all (recurse to all), \
--recurse-not-to=%(full_path)s (ignore it), \
--recurse-to=%(full_path)s (recurse to it) to change.""" % {
"full_path": module_fullpath,
"filename": module_filename
})
def runScons(options, quiet):
# For the scons file to find the static C++ files and include path. The
# scons file is unable to use __file__ for the task.
os.environ["NUITKA_SCONS"] = getSconsDataPath()
if Utils.getOS() == "Windows":
# On Windows this Scons variable must be set by us.
os.environ["SCONS_LIB_DIR"] = Utils.joinpath(
getSconsInlinePath(),
"lib",
"scons-2.3.2"
)
scons_command = getSconsBinaryCall()
if quiet:
scons_command.append("--quiet")
scons_command += [
# The scons file
"-f",
Utils.joinpath(getSconsDataPath(), "SingleExe.scons"),
# Parallel compilation.
"--jobs",
str(Options.getJobLimit()),
# Do not warn about deprecations of Scons
"--warn=no-deprecated",
# Don't load "site_scons" at all.
"--no-site-dir",
]
if Options.isShowScons():
scons_command.append("--debug=explain")
# Option values to provide to scons.
for key, value in options.items():
scons_command += [key + "=" + value]
if Options.isShowScons():
Tracing.printLine("Scons command:", " ".join(scons_command))
return 0 == subprocess.call(scons_command, shell = False)
def checkPluginPath( plugin_filename, module_package ):
plugin_info = considerFilename(
module_package = module_package,
module_filename = plugin_filename
)
if plugin_info is not None:
# File or package, handle that.
if Utils.isFile( plugin_info[0] ) or Importing.isPackageDir( plugin_info[0] ):
_checkPluginPath( plugin_filename, module_package )
elif Utils.isDir( plugin_info[0] ):
for sub_path, sub_filename in Utils.listDir( plugin_info[0] ):
assert sub_filename != "__init__.py"
if Importing.isPackageDir( sub_path ) or sub_path.endswith( ".py" ):
_checkPluginPath( sub_path, None )
else:
warning( "Failed to include module from '%s'.", plugin_info[0] )
def getImportedModule(module_relpath):
module_name = Utils.basename(module_relpath)
if module_name.endswith(".py"):
module_name = module_name[:-3]
key = module_relpath, module_name
return imported_modules[key]
def isImportedPath(module_relpath):
module_name = Utils.basename(module_relpath)
if module_name.endswith(".py"):
module_name = module_name[:-3]
key = module_relpath, module_name
return key in imported_modules
def checkPluginPath(plugin_filename, module_package):
plugin_info = considerFilename(module_package=module_package,
module_filename=plugin_filename)
if plugin_info is not None:
# File or package, handle that.
if Utils.isFile(plugin_info[0]) or Importing.isPackageDir(
plugin_info[0]):
_checkPluginPath(plugin_filename, module_package)
elif Utils.isDir(plugin_info[0]):
for sub_path, sub_filename in Utils.listDir(plugin_info[0]):
assert sub_filename != "__init__.py"
if Importing.isPackageDir(sub_path) or sub_path.endswith(
".py"):
_checkPluginPath(sub_path, None)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
def __init__(self, name, package_name, source_ref):
NodeBase.__init__(self, source_ref=source_ref)
PythonModuleMixin.__init__(self, name=name, package_name=package_name)
assert Utils.basename(source_ref.getFilename()) != "<frozen>"
# That is too likely a bug.
assert name != "__main__"
def getImportedModule( module_relpath ):
module_name = Utils.basename( module_relpath )
if module_name.endswith( ".py" ):
module_name = module_name[:-3]
key = module_relpath, module_name
return imported_modules[ key ]
def isImportedPath( module_relpath ):
module_name = Utils.basename( module_relpath )
if module_name.endswith( ".py" ):
module_name = module_name[:-3]
key = module_relpath, module_name
return key in imported_modules
def _consider( self, constraint_collection, module_filename, module_package ):
assert module_package is None or ( type( module_package ) is str and module_package != "" )
module_filename = Utils.normpath( module_filename )
if Utils.isDir( module_filename ):
module_name = Utils.basename( module_filename )
elif module_filename.endswith( ".py" ):
module_name = Utils.basename( module_filename )[:-3]
else:
module_name = None
if module_name is not None:
decision = self._decide( module_filename, module_name, module_package )
if decision:
module_relpath = Utils.relpath( module_filename )
return self._recurseTo(
constraint_collection = constraint_collection,
module_package = module_package,
module_filename = module_filename,
module_relpath = module_relpath
)
elif decision is None:
if module_package is None:
module_fullpath = module_name
else:
module_fullpath = module_package + "." + module_name
if module_filename not in self._warned_about:
self._warned_about.add( module_filename )
warning( # long message, but shall be like it, pylint: disable=C0301
"""\
Not recursing to '%(full_path)s' (%(filename)s), please specify \
--recurse-none (do not warn), \
--recurse-all (recurse to all), \
--recurse-not-to=%(full_path)s (ignore it), \
--recurse-to=%(full_path)s (recurse to it) to change.""" % {
"full_path" : module_fullpath,
"filename" : module_filename
}
)
def detectBinaryDLLs(binary_filename, package_name):
""" Detect the DLLs used by a binary.
Using ldd (Linux), depends.exe (Windows), or otool (MacOS) the list
of used DLLs is retrieved.
"""
if Utils.getOS() in ("Linux", "NetBSD"):
# TODO: FreeBSD may work the same way, not tested.
return _detectBinaryPathDLLsLinuxBSD(binary_filename=binary_filename)
elif Utils.getOS() == "Windows":
return _detectBinaryPathDLLsWindows(binary_filename=binary_filename,
package_name=package_name)
elif Utils.getOS() == "Darwin":
return _detectBinaryPathDLLsLinuxBSD(binary_filename=binary_filename)
else:
# Support your platform above.
assert False, Utils.getOS()
def _makeBinaryPathPathDLLSearchEnv(package_name):
# Put the PYTHONPATH into the system "PATH", DLLs frequently live in
# the package directories.
env = os.environ.copy()
path = env.get("PATH", "").split(';')
# Put the "Python.exe" first. At least for WinPython, they put the DLLs
# there.
path = [sys.prefix] + sys.path + path
if package_name is not None:
for element in sys.path:
candidate = Utils.joinpath(element, package_name)
if Utils.isDir(candidate):
path.append(candidate)
env["PATH"] = ';'.join(path)
return env
def runScons( options, quiet ):
# For the scons file to find the static C++ files and include path. The scons file is
# unable to use __file__ for the task.
os.environ[ "NUITKA_SCONS" ] = getSconsDataPath()
if os.name == "nt":
# On Windows this Scons variable must be set by us.
os.environ[ "SCONS_LIB_DIR" ] = Utils.joinpath( getSconsInlinePath(), "lib", "scons-2.0.1" )
# Also, for MinGW we can avoid the user having to add the path if he used the
# default path or installed it on the same drive by appending to the PATH variable
# before executing scons.
os.environ[ "PATH" ] += r";\MinGW\bin;C:\MinGW\bin"
# Scons is Python2 only, so we need to make the system find a suitable Python binary.
if Utils.python_version < 300:
python_exe = sys.executable
elif os.name == "nt":
if os.path.exists( r"c:\Python27\python.exe" ):
python_exe = r"c:\Python27\python.exe"
elif os.path.exists( r"c:\Python26\python.exe" ):
python_exe = r"c:\Python26\python.exe"
else:
sys.exit( """Error, need to find Python2 executable under C:\\Python26 or \
C:\\Python27 to execute scons which is not Python3 compatible.""" )
else:
python_exe = "python"
scons_command = """%(python)s %(binary)s %(quiet)s --warn=no-deprecated -f %(scons_file)s --jobs %(job_limit)d %(options)s""" % {
"python" : python_exe,
"binary" : getSconsBinaryPath(),
"quiet" : "--quiet" if quiet else "",
"scons_file" : Utils.joinpath( getSconsDataPath(), "SingleExe.scons" ),
"job_limit" : Options.getJobLimit(),
"options" : " ".join( "%s=%s" % ( key, value ) for key, value in options.items() )
}
if Options.isShowScons():
Tracing.printLine( "Scons command:", scons_command )
return 0 == os.system( scons_command )
def getMainCode(codes, context):
python_flags = Options.getPythonFlags()
if context.isEmptyModule():
code_identifier = NullIdentifier()
else:
code_identifier = getCodeObjectHandle(
context = context,
filename = context.getFilename(),
var_names = (),
arg_count = 0,
kw_only_count = 0,
line_number = 0,
code_name = "<module>",
is_generator = False,
is_optimized = False,
has_starlist = False,
has_stardict = False
)
main_code = CodeTemplates.main_program % {
"sys_executable" : getConstantCode(
constant = "python.exe"
if Utils.getOS() == "Windows" and \
Options.isStandaloneMode() else
sys.executable,
context = context
),
"python_sysflag_debug" : sys.flags.debug,
"python_sysflag_py3k_warning" : ( sys.flags.py3k_warning
if hasattr( sys.flags, "py3k_warning" ) else 0 ),
"python_sysflag_division_warning" : ( sys.flags.division_warning
if hasattr( sys.flags, "division_warning" ) else 0 ),
#"python_sysflag_division_new" : sys.flags.division_new, #not supported
"python_sysflag_inspect" : sys.flags.inspect,
"python_sysflag_interactive" : sys.flags.interactive,
"python_sysflag_optimize" : sys.flags.optimize,
"python_sysflag_dont_write_bytecode" : sys.flags.dont_write_bytecode,
"python_sysflag_no_site" : sys.flags.no_site,
"python_sysflag_no_user_site" : sys.flags.no_user_site,
"python_sysflag_ignore_environment" : sys.flags.ignore_environment,
"python_sysflag_tabcheck" : ( sys.flags.tabcheck
if hasattr( sys.flags, "tabcheck" ) else 0 ),
"python_sysflag_verbose" : 1 if "trace_imports" in python_flags else 0,
"python_sysflag_unicode" : ( sys.flags.unicode
if hasattr( sys.flags, "unicode" ) else 0 ),
"python_sysflag_bytes_warning" : sys.flags.bytes_warning,
"python_sysflag_hash_randomization" : ( sys.flags.hash_randomization
if hasattr( sys.flags, "hash_randomization" ) and "no_randomization" not in python_flags else 0 ),
"code_identifier" : code_identifier.getCodeTemporaryRef()
}
return codes + main_code
def isSameModulePath(path1, path2):
if Utils.basename(path1) == "__init__.py":
path1 = Utils.dirname(path1)
if Utils.basename(path2) == "__init__.py":
path2 = Utils.dirname(path2)
return Utils.abspath(path1) == Utils.abspath(path2)
def isSameModulePath(path1, path2):
if Utils.basename(path1) == "__init__.py":
path1 = Utils.dirname(path1)
if Utils.basename(path2) == "__init__.py":
path2 = Utils.dirname(path2)
return Utils.abspath(path1) == Utils.abspath(path2)
def __init__(self, name, package_name, source_ref):
NodeBase.__init__(
self,
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
assert Utils.basename(source_ref.getFilename()) != "<frozen>"
def scanStandardLibraryPath(stdlib_dir):
# There is a lot of black-listing here, done in branches, so there
# is many of them, but that's acceptable, pylint: disable=R0912
for root, dirs, filenames in os.walk(stdlib_dir):
import_path = root[len(stdlib_dir):].strip('/\\')
import_path = import_path.replace("\\", ".").replace("/",".")
if import_path == '':
if 'site-packages' in dirs:
dirs.remove('site-packages')
if 'dist-packages' in dirs:
dirs.remove('dist-packages')
if 'test' in dirs:
dirs.remove('test')
if 'idlelib' in dirs:
dirs.remove('idlelib')
if 'turtledemo' in dirs:
dirs.remove('turtledemo')
if import_path in ('tkinter', 'importlib', 'ctypes'):
if 'test' in dirs:
dirs.remove('test')
if import_path == "lib2to3":
if 'tests' in dirs:
dirs.remove('tests')
if Utils.python_version >= 340 and Utils.getOS() == "Windows":
if import_path == "multiprocessing":
filenames.remove("popen_fork.py")
filenames.remove("popen_forkserver.py")
filenames.remove("popen_spawn_posix.py")
for filename in filenames:
if filename.endswith('.py') and filename not in ignore_modules:
module_name = filename[:-3]
if import_path == '':
yield module_name
else:
yield import_path + '.' + module_name
if Utils.python_version >= 300:
if '__pycache__' in dirs:
dirs.remove('__pycache__')
for dirname in dirs:
if import_path == '':
yield dirname
else:
yield import_path + '.' + dirname
def _checkPluginPath( plugin_filename, module_package ):
plugin_info = considerFilename(
module_package = module_package,
module_filename = plugin_filename
)
if plugin_info is not None:
module, added = recurseTo(
module_filename = plugin_info[0],
module_relpath = plugin_info[1],
module_package = module_package
)
if module:
if not added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isPythonPackage() else "module",
module.getName(),
plugin_info[0]
)
if module.isPythonPackage():
package_dir = Utils.dirname( module.getFilename() )
for sub_path, sub_filename in Utils.listDir( package_dir ):
if sub_filename == "__init__.py":
continue
assert sub_path != plugin_filename, package_dir
if Importing.isPackageDir( sub_path ) or sub_path.endswith( ".py" ):
_checkPluginPath( sub_path, module.getFullName() )
else:
warning( "Failed to include module from '%s'.", plugin_info[0] )
def __init__(self, name, package_name, source_ref):
NodeBase.__init__(
self,
source_ref = source_ref
)
PythonModuleMixin.__init__(
self,
name = name,
package_name = package_name
)
assert Utils.basename(source_ref.getFilename()) != "<frozen>"
# That is too likely a bug.
assert name != "__main__"
def _detectedSourceFile(filename, module_name, result, is_late):
if module_name in module_names:
return
if module_name == "collections.abc":
_detectedSourceFile(
filename = filename,
module_name = "_collections_abc",
result = result,
is_late = is_late
)
source_code = open(filename,"rb").read()
if Utils.python_version >= 300:
source_code = source_code.decode("utf-8")
filename = filename.decode("utf-8")
if module_name == "site":
source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
os.path.sep,
os.path.basename(filename),
source_code
)
debug(
"Freezing module '%s' (from '%s').",
module_name,
filename
)
result.append(
(
module_name,
marshal.dumps(
compile(source_code, filename, "exec")
),
Utils.basename(filename) == "__init__.py",
filename,
is_late
)
)
module_names.add(module_name)
def getPython2ExePath():
""" Find a way to call Python2. Scons needs it."""
if Utils.python_version < 300:
return sys.executable
elif Utils.getOS() == "Windows":
python_exe = _getPython2ExePathWindows()
if python_exe is not None:
return python_exe
else:
sys.exit("""\
Error, need to find Python2 executable under C:\\Python26 or \
C:\\Python27 to execute scons which is not Python3 compatible.""")
elif os.path.exists("/usr/bin/python2"):
return "python2"
else:
return "python"
def attemptRecursion(self):
# Make sure the package is recursed to.
from nuitka.tree import Recursion
from nuitka import Importing
# Return the list of newly added modules.
result = []
if self.package_name is not None and self.package is None:
package_package, _package_module_name, package_filename = \
Importing.findModule(
source_ref = self.getSourceReference(),
module_name = self.package_name,
parent_package = None,
level = 1,
warn = Utils.python_version < 330
)
# TODO: Temporary, if we can't find the package for Python3.3 that
# is semi-OK, maybe.
if Utils.python_version >= 330 and not package_filename:
return []
imported_module, is_added = Recursion.recurseTo(
module_package = package_package,
module_filename = package_filename,
module_relpath = Utils.relpath(package_filename),
module_kind = "py",
reason = "Containing package of recursed module.",
)
self.package = imported_module
if is_added:
result.append(imported_module)
if self.package:
from nuitka.ModuleRegistry import addUsedModule
addUsedModule(self.package)
# print "Recursed to package", self.package_name
result.extend(self.package.attemptRecursion())
return result
def copyUsedDLLs(dist_dir, binary_filename, standalone_entry_points):
dll_map = []
for early_dll in detectUsedDLLs(standalone_entry_points):
dll_name = Utils.basename(early_dll)
target_path = Utils.joinpath(
dist_dir,
dll_name
)
# Check that if a DLL has the name name, if it's identical,
# happens at least for OSC and Fedora 20.
if Utils.isFile(target_path):
import filecmp
if filecmp.cmp(early_dll, target_path):
continue
else:
sys.exit("Error, conflicting DLLs for '%s'." % dll_name)
shutil.copy(
early_dll,
target_path
)
dll_map.append(
(early_dll, dll_name)
)
if Options.isShowInclusion():
info("Included used shared library '%s'.", early_dll)
if Utils.getOS() == "Darwin":
# For MacOS, the binary needs to be changed to reflect the DLL
# location in the dist folder.
fixupBinaryDLLPaths(binary_filename, dll_map)
if Utils.getOS() == "Linux":
# For Linux, the rpath of libraries may be an issue.
for _original_path, early_dll in dll_map:
removeSharedLibraryRPATH(
Utils.joinpath(dist_dir, early_dll)
)
for standalone_entry_point in standalone_entry_points[1:]:
removeSharedLibraryRPATH(
standalone_entry_point[0]
)
