def isSameModulePath(path1, path2):
if Utils.basename(path1) == "__init__.py":
path1 = Utils.dirname(path1)
if Utils.basename(path2) == "__init__.py":
path2 = Utils.dirname(path2)
return Utils.abspath(path1) == Utils.abspath(path2)
def isSameModulePath(path1, path2):
if Utils.basename(path1) == "__init__.py":
path1 = Utils.dirname(path1)
if Utils.basename(path2) == "__init__.py":
path2 = Utils.dirname(path2)
return Utils.abspath(path1) == Utils.abspath(path2)
def detectEarlyImports():
encoding_names = [
filename[:-3] for _path, filename in Utils.listDir(
Utils.dirname(sys.modules["encodings"].__file__))
if filename.endswith(".py") if "__init__" not in filename
]
if Utils.getOS() != "Windows":
encoding_names.remove("mbcs")
if "cp65001" in encoding_names:
encoding_names.remove("cp65001")
import_code = ';'.join("import encodings.%s" % encoding_name
for encoding_name in encoding_names)
import_code += ";import locale;"
# For Python3 we patch inspect without knowing if it is used.
if python_version >= 300:
import_code += "import inspect;"
# We might need the pickle module when creating global constants.
if python_version >= 300:
import_code += "import pickle;"
else:
import_code += "import cPickle;"
result = _detectImports(command=import_code,
user_provided=False,
technical=True)
if Options.shallFreezeAllStdlib():
stdlib_modules = set()
# Scan the standard library paths (multiple in case of virtualenv.
for stdlib_dir in getStandardLibraryPaths():
for module_name in scanStandardLibraryPath(stdlib_dir):
stdlib_modules.add(module_name)
import_code = "imports = " + repr(sorted(stdlib_modules)) + '\n'\
"for imp in imports:\n" \
" try:\n" \
" __import__(imp)\n" \
" except (ImportError, SyntaxError):\n" \
" pass\n"
early_names = [module.getFullName() for module in result]
result += [
module for module in _detectImports(
command=import_code, user_provided=False, technical=False)
if module.getFullName() not in early_names
]
debug("Finished detecting early imports.")
return result
def __init__(self, *args):
QtGui.QDialog.__init__(self, *args)
ui_dir = Utils.dirname(__file__)
ui_filename = Utils.joinpath(ui_dir, "dialogs", "InspectPythonTree.ui")
uic.loadUi(ui_filename, self)
self.treeview_nodes.setSelectionMode(self.treeview_nodes.SingleSelection)
self.displayed = None
self.source_code = None
self.model = None
self.moving = None
def __init__(self, *args):
QtGui.QDialog.__init__(self, *args)
ui_dir = Utils.dirname(__file__)
ui_filename = Utils.joinpath(ui_dir, "dialogs", "InspectPythonTree.ui")
uic.loadUi(ui_filename, self)
self.treeview_nodes.setSelectionMode(self.treeview_nodes.SingleSelection)
self.displayed = None
self.source_code = None
self.model = None
self.moving = None
def _normalizeModuleFilename(filename):
if python_version >= 300:
filename = filename.replace("__pycache__", "")
suffix = ".cpython-%d.pyc" % (python_version // 10)
if filename.endswith(suffix):
filename = filename[:-len(suffix)] + ".py"
else:
if filename.endswith(".pyc"):
filename = filename[:-3] + ".py"
if Utils.basename(filename) == "__init__.py":
filename = Utils.dirname(filename)
return filename
def _normalizeModuleFilename(filename):
if python_version >= 300:
filename = filename.replace("__pycache__", "")
suffix = ".cpython-%d.pyc" % (python_version // 10)
if filename.endswith(suffix):
filename = filename[:-len(suffix)] + ".py"
else:
if filename.endswith(".pyc"):
filename = filename[:-3] + ".py"
if Utils.basename(filename) == "__init__.py":
filename = Utils.dirname(filename)
return filename
def addImportedModule(imported_module):
module_filename = Utils.relpath(imported_module.getFilename())
if Utils.basename(module_filename) == "__init__.py":
module_filename = Utils.dirname(module_filename)
key = (module_filename, imported_module.getFullName())
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
# We don't expect that to happen.
assert not imported_module.isMainModule()
def createPathAssignment(source_ref):
if Options.getFileReferenceMode() == "original":
path_value = ExpressionConstantRef(
constant = [
Utils.dirname(source_ref.getFilename())
],
source_ref = source_ref,
user_provided = True
)
else:
path_value = ExpressionMakeList(
elements = (
ExpressionCallNoKeywords(
called = ExpressionAttributeLookup(
source = ExpressionImportModuleHard(
module_name = "os",
import_name = "path",
source_ref = source_ref
),
attribute_name = "dirname",
source_ref = source_ref
),
args = ExpressionMakeTuple(
elements = (
ExpressionModuleFileAttributeRef(
source_ref = source_ref,
),
),
source_ref = source_ref,
),
source_ref = source_ref,
),
),
source_ref = source_ref
)
return StatementAssignmentVariable(
variable_ref = ExpressionTargetVariableRef(
variable_name = "__path__",
source_ref = source_ref
),
source = path_value,
source_ref = source_ref
)
def addImportedModule(imported_module):
module_filename = Utils.relpath(imported_module.getFilename())
if Utils.basename(module_filename) == "__init__.py":
module_filename = Utils.dirname(module_filename)
key = (
module_filename,
imported_module.getFullName()
)
if key in imported_modules:
assert imported_module is imported_modules[key], key
else:
Plugins.onModuleDiscovered(imported_module)
imported_modules[key] = imported_module
imported_by_name[imported_module.getFullName()] = imported_module
# We don't expect that to happen.
assert not imported_module.isMainModule()
def getRunTimeFilename(self):
reference_mode = Options.getFileReferenceMode()
if reference_mode == "original":
return self.getCompileTimeFilename()
elif reference_mode == "frozen":
return "<frozen %s>" % self.getFullName()
else:
filename = self.getCompileTimeFilename()
full_name = self.getFullName()
result = Utils.basename(filename)
current = filename
levels = full_name.count('.')
if self.isCompiledPythonPackage():
levels += 1
for _i in range(levels):
current = Utils.dirname(current)
result = Utils.joinpath(Utils.basename(current), result)
return result
def getRunTimeFilename(self):
reference_mode = Options.getFileReferenceMode()
if reference_mode == "original":
return self.getCompileTimeFilename()
elif reference_mode == "frozen":
return "<frozen %s>" % self.getFullName()
else:
filename = self.getCompileTimeFilename()
full_name = self.getFullName()
result = Utils.basename(filename)
current = filename
levels = full_name.count('.')
if self.isCompiledPythonPackage():
levels += 1
for _i in range(levels):
current = Utils.dirname(current)
result = Utils.joinpath(Utils.basename(current), result)
return result
def makeSourceDirectory(main_module):
""" Get the full list of modules imported, create code for all of them.
"""
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=R0912,R0914
assert main_module.isCompiledPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
assert main_module in ModuleRegistry.getDoneModules()
# We might have chosen to include it as bytecode, and only compiled it for
# fun, and to find its imports. In this case, now we just can drop it. Or
# a module may shadow a frozen module, but be a different one, then we can
# drop the frozen one.
# TODO: This really should be done when the compiled module comes into
# existence.
for module in ModuleRegistry.getDoneUserModules():
if module.isCompiledPythonModule():
uncompiled_module = ModuleRegistry.getUncompiledModule(
module_name = module.getFullName(),
module_filename = module.getCompileTimeFilename()
)
if uncompiled_module is not None:
# We now need to decide which one to keep, compiled or uncompiled
# module. Some uncompiled modules may have been asked by the user
# or technically required. By default, frozen code if it exists
# is preferred, as it will be from standalone mode adding it.
if uncompiled_module.isUserProvided():
ModuleRegistry.removeDoneModule(module)
else:
ModuleRegistry.removeUncompiledModule(uncompiled_module)
# Lets check if the recurse-to modules are actually present, and warn the
# user if one of those was not found.
for any_case_module in Options.getShallFollowModules():
for module in ModuleRegistry.getDoneUserModules():
if module.getFullName() == any_case_module:
break
else:
warning(
"Didn't recurse to '%s', apparently not used." % \
any_case_module
)
# Prepare code generation, i.e. execute finalization for it.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = getSourceDirectoryPath(main_module)
module_filenames = pickSourceFilenames(
source_dir = source_dir,
modules = ModuleRegistry.getDoneModules()
)
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
cpp_filename = module_filenames[module]
prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode(
global_context = global_context,
module = module,
module_name = module.getFullName(),
)
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[cpp_filename][1].getConstantCode(0)
# Second pass, generate the actual module code into the files.
for module in ModuleRegistry.getDoneModules():
if module.isCompiledPythonModule():
cpp_filename = module_filenames[module]
template_values, module_context = prepared_modules[cpp_filename]
source_code = CodeGeneration.generateModuleCode(
module_context = module_context,
template_values = template_values
)
# The main of an executable module gets a bit different code.
if module is main_module and not Options.shallMakeModule():
source_code = MainCodes.generateMainCode(
main_module = main_module,
context = module_context,
codes = source_code
)
writeSourceCode(
filename = cpp_filename,
source_code = source_code
)
if Options.isShowInclusion():
info("Included compiled module '%s'." % module.getFullName())
elif module.isPythonShlibModule():
target_filename = Utils.joinpath(
getStandaloneDirectoryPath(main_module),
*module.getFullName().split('.')
)
if Utils.getOS() == "Windows":
target_filename += ".pyd"
else:
target_filename += ".so"
target_dir = Utils.dirname(target_filename)
if not Utils.isDir(target_dir):
Utils.makePath(target_dir)
shutil.copy(
module.getFilename(),
target_filename
)
standalone_entry_points.append(
(target_filename, module.getPackage())
)
else:
assert False, module
writeSourceCode(
filename = Utils.joinpath(source_dir, "__constants.cpp"),
source_code = ConstantCodes.getConstantsDefinitionCode(
context = global_context
)
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode(
ModuleRegistry.getDoneUserModules()
)
writeSourceCode(
filename = Utils.joinpath(source_dir, "__helpers.hpp"),
source_code = helper_decl_code
)
writeSourceCode(
filename = Utils.joinpath(source_dir, "__helpers.cpp"),
source_code = helper_impl_code
)
def detectEarlyImports():
encoding_names = [
filename[:-3]
for _path, filename in
Utils.listDir(Utils.dirname(sys.modules["encodings"].__file__))
if filename.endswith(".py")
if "__init__" not in filename
]
if Utils.getOS() != "Windows":
encoding_names.remove("mbcs")
if "cp65001" in encoding_names:
encoding_names.remove("cp65001")
import_code = ";".join(
"import encodings.%s" % encoding_name
for encoding_name in
encoding_names
)
import_code += ";import locale;"
# For Python3 we patch inspect without knowing if it is used.
if python_version >= 300:
import_code += "import inspect;"
# We might need the pickle module when creating global constants.
if python_version >= 300:
import_code += "import pickle;"
else:
import_code += "import cPickle;"
result = _detectImports(
command = import_code,
user_provided = False,
technical = True
)
if Options.shallFreezeAllStdlib():
stdlib_modules = set()
# Scan the standard library paths (multiple in case of virtualenv.
for stdlib_dir in getStandardLibraryPaths():
for module_name in scanStandardLibraryPath(stdlib_dir):
stdlib_modules.add(module_name)
import_code = "imports = " + repr(sorted(stdlib_modules)) + '\n'\
"for imp in imports:\n" \
" try:\n" \
" __import__(imp)\n" \
" except (ImportError, SyntaxError):\n" \
" pass\n"
early_names = [
module.getFullName()
for module in result
]
result += [
module
for module in _detectImports(
command = import_code,
user_provided = False,
technical = False
)
if module.getFullName() not in early_names
]
debug("Finished detecting early imports.")
return result
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex
debug(
"Checking detail plug-in path '%s' '%s':",
plugin_filename,
module_package
)
plugin_info = considerFilename(
module_filename = plugin_filename
)
if plugin_info is not None:
module, is_added = recurseTo(
module_filename = plugin_info[0],
module_relpath = plugin_info[1],
module_package = module_package,
module_kind = "py",
reason = "Lives in plug-in directory."
)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isCompiledPythonPackage() else "module",
module.getName(),
plugin_info[0]
)
if not isSameModulePath(module.getFilename(), plugin_info[0]):
warning(
"Duplicate ignored '%s'.",
plugin_info[1]
)
return
debug(
"Recursed to %s %s %s",
module.getName(),
module.getPackage(),
module
)
ImportCache.addImportedModule(module)
if module.isCompiledPythonPackage():
package_filename = module.getFilename()
if Utils.isDir(package_filename):
# Must be a namespace package.
assert python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
else:
package_dir = Utils.dirname(package_filename)
# Real packages will always be included.
ModuleRegistry.addRootModule(module)
debug(
"Package directory %s",
package_dir
)
for sub_path, sub_filename in Utils.listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
elif module.isCompiledPythonModule():
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
def makeSourceDirectory(main_module):
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=R0914,R0912
assert main_module.isPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
# Get the full list of modules imported, create code for all of them.
modules = ModuleRegistry.getDoneModules()
assert main_module in modules
# Sometimes we need to talk about all modules except main module.
other_modules = ModuleRegistry.getDoneUserModules()
# Lets check if the recurse-to modules are actually present, and warn the
# user if one was not found.
for any_case_module in Options.getShallFollowModules():
for module in other_modules:
if module.getFullName() == any_case_module:
break
else:
warning(
"Didn't recurse to '%s', apparently not used." % \
any_case_module
)
# Prepare code generation, i.e. execute finalization for it.
for module in sorted(modules, key=lambda x: x.getFullName()):
if module.isPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = getSourceDirectoryPath(main_module)
module_filenames = pickSourceFilenames(source_dir=source_dir,
modules=modules)
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in sorted(modules, key=lambda x: x.getFullName()):
if module.isPythonModule():
cpp_filename = module_filenames[module]
prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode(
global_context=global_context,
module=module,
module_name=module.getFullName(),
other_modules=other_modules if module is main_module else ())
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[cpp_filename][1].getConstantCode(0)
for module in sorted(modules, key=lambda x: x.getFullName()):
if module.isPythonModule():
cpp_filename = module_filenames[module]
template_values, module_context = prepared_modules[cpp_filename]
source_code = CodeGeneration.generateModuleCode(
module_context=module_context, template_values=template_values)
# The main of an executable module gets a bit different code.
if module is main_module and not Options.shallMakeModule():
source_code = CodeGeneration.generateMainCode(
main_module=main_module,
context=module_context,
codes=source_code)
writeSourceCode(filename=cpp_filename, source_code=source_code)
if Options.isShowInclusion():
info("Included compiled module '%s'." % module.getFullName())
elif module.isPythonShlibModule():
target_filename = Utils.joinpath(
getStandaloneDirectoryPath(main_module),
*module.getFullName().split('.'))
if Utils.getOS() == "Windows":
target_filename += ".pyd"
else:
target_filename += ".so"
target_dir = Utils.dirname(target_filename)
if not Utils.isDir(target_dir):
Utils.makePath(target_dir)
shutil.copy(module.getFilename(), target_filename)
standalone_entry_points.append(
(target_filename, module.getPackage()))
else:
assert False, module
writeSourceCode(filename=Utils.joinpath(source_dir, "__constants.cpp"),
source_code=ConstantCodes.getConstantsDefinitionCode(
context=global_context))
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode()
writeSourceCode(filename=Utils.joinpath(source_dir, "__helpers.hpp"),
source_code=helper_decl_code)
writeSourceCode(filename=Utils.joinpath(source_dir, "__helpers.cpp"),
source_code=helper_impl_code)
def makeSourceDirectory(main_module):
# We deal with a lot of details here, but rather one by one, and split makes
# no sense, pylint: disable=R0914,R0912
assert main_module.isPythonModule()
# The global context used to generate code.
global_context = CodeGeneration.makeGlobalContext()
# Get the full list of modules imported, create code for all of them.
modules = ModuleRegistry.getDoneModules()
assert main_module in modules
# Sometimes we need to talk about all modules except main module.
other_modules = ModuleRegistry.getDoneUserModules()
# Lets check if the recurse-to modules are actually present, and warn the
# user if one was not found.
for any_case_module in Options.getShallFollowModules():
for module in other_modules:
if module.getFullName() == any_case_module:
break
else:
warning(
"Didn't recurse to '%s', apparently not used." % \
any_case_module
)
# Prepare code generation, i.e. execute finalization for it.
for module in sorted(modules, key = lambda x : x.getFullName()):
if module.isPythonModule():
Finalization.prepareCodeGeneration(module)
# Pick filenames.
source_dir = getSourceDirectoryPath(main_module)
module_filenames = pickSourceFilenames(
source_dir = source_dir,
modules = modules
)
# First pass, generate code and use constants doing so, but prepare the
# final code generation only, because constants code will be added at the
# end only.
prepared_modules = {}
for module in sorted(modules, key = lambda x : x.getFullName()):
if module.isPythonModule():
cpp_filename = module_filenames[module]
prepared_modules[cpp_filename] = CodeGeneration.prepareModuleCode(
global_context = global_context,
module = module,
module_name = module.getFullName(),
other_modules = other_modules
if module is main_module else
()
)
# Main code constants need to be allocated already too.
if module is main_module and not Options.shallMakeModule():
prepared_modules[cpp_filename][1].getConstantCode(0)
for module in sorted(modules, key = lambda x : x.getFullName()):
if module.isPythonModule():
cpp_filename = module_filenames[module]
template_values, module_context = prepared_modules[cpp_filename]
source_code = CodeGeneration.generateModuleCode(
module_context = module_context,
template_values = template_values
)
# The main of an executable module gets a bit different code.
if module is main_module and not Options.shallMakeModule():
source_code = CodeGeneration.generateMainCode(
main_module = main_module,
context = module_context,
codes = source_code
)
writeSourceCode(
filename = cpp_filename,
source_code = source_code
)
if Options.isShowInclusion():
info("Included compiled module '%s'." % module.getFullName())
elif module.isPythonShlibModule():
target_filename = Utils.joinpath(
getStandaloneDirectoryPath(main_module),
*module.getFullName().split('.')
)
if Utils.getOS() == "Windows":
target_filename += ".pyd"
else:
target_filename += ".so"
target_dir = Utils.dirname(target_filename)
if not Utils.isDir(target_dir):
Utils.makePath(target_dir)
shutil.copy(
module.getFilename(),
target_filename
)
standalone_entry_points.append(
(target_filename, module.getPackage())
)
else:
assert False, module
writeSourceCode(
filename = Utils.joinpath(source_dir, "__constants.cpp"),
source_code = ConstantCodes.getConstantsDefinitionCode(
context = global_context
)
)
helper_decl_code, helper_impl_code = CodeGeneration.generateHelpersCode()
writeSourceCode(
filename = Utils.joinpath(source_dir, "__helpers.hpp"),
source_code = helper_decl_code
)
writeSourceCode(
filename = Utils.joinpath(source_dir, "__helpers.cpp"),
source_code = helper_impl_code
)
def getOutputFilename(self):
if self.main_added:
return Utils.dirname(self.getFilename())
else:
return CompiledPythonModule.getOutputFilename(self)
def _detectImports(command, is_late):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=R0912,R0914
# Print statements for stuff to show, the modules loaded.
if Utils.python_version >= 300:
command += '\nimport sys\nprint("\\n".join(sorted("import " + module.__name__ + " # sourcefile " + ' \
'module.__file__ for module in sys.modules.values() if hasattr(module, "__file__") and ' \
'module.__file__ != "<frozen>")), file = sys.stderr)' # do not read it
reduced_path = [
path_element
for path_element in
sys.path
if not Utils.areSamePaths(
path_element,
'.'
)
if not Utils.areSamePaths(
path_element,
Utils.dirname(sys.modules["__main__"].__file__)
)
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s;" % repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if Utils.python_version >= 300:
command = command.encode("ascii")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args = [sys.executable, "-s", "-S", "-v", tmp_filename],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
warning("There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
Tracing.printLine(line)
sys.exit("Error, please report the issue with above output.")
result = []
debug("Detecting imports:")
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if Utils.python_version >= 300:
module_name = module_name.decode("utf-8")
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
_detectedPrecompiledFile(
filename = filename,
module_name = module_name,
result = result,
is_late = is_late
)
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if filename.endswith(b".py"):
_detectedSourceFile(
filename = filename,
module_name = module_name,
result = result,
is_late = is_late
)
elif not filename.endswith(b"<frozen>"):
_detectedShlibFile(
filename = filename,
module_name = module_name
)
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
_detectedShlibFile(
filename = filename,
module_name = module_name
)
return result
def getStandardLibraryPaths():
""" Get the standard library paths.
"""
# Using the function object to cache its result, avoiding global variable
# usage.
if not hasattr(getStandardLibraryPaths, "result"):
os_filename = os.__file__
if os_filename.endswith(".pyc"):
os_filename = os_filename[:-1]
os_path = Utils.normcase(Utils.dirname(os_filename))
stdlib_paths = set([os_path])
# Happens for virtualenv situation, some modules will come from the link
# this points to.
if Utils.isLink(os_filename):
os_filename = Utils.readLink(os_filename)
stdlib_paths.add(Utils.normcase(Utils.dirname(os_filename)))
# Another possibility is "orig-prefix.txt" file near the os.py, which
# points to the original install.
orig_prefix_filename = Utils.joinpath(os_path, "orig-prefix.txt")
if Utils.isFile(orig_prefix_filename):
# Scan upwards, until we find a "bin" folder, with "activate" to
# locate the structural path to be added. We do not know for sure
# if there is a sub-directory under "lib" to use or not. So we try
# to detect it.
search = os_path
lib_part = ""
while os.path.splitdrive(search)[1] not in (os.path.sep, ""):
if Utils.isFile(Utils.joinpath(search,"bin/activate")) or \
Utils.isFile(Utils.joinpath(search,"scripts/activate")):
break
lib_part = Utils.joinpath(Utils.basename(search), lib_part)
search = Utils.dirname(search)
assert search and lib_part
stdlib_paths.add(
Utils.normcase(
Utils.joinpath(
open(orig_prefix_filename).read(),
lib_part,
)))
# And yet another possibility, for MacOS Homebrew created virtualenv
# at least is a link ".Python", which points to the original install.
python_link_filename = Utils.joinpath(os_path, "..", ".Python")
if Utils.isLink(python_link_filename):
stdlib_paths.add(
Utils.normcase(
Utils.joinpath(Utils.readLink(python_link_filename),
"lib")))
getStandardLibraryPaths.result = [
Utils.normcase(stdlib_path) for stdlib_path in stdlib_paths
]
return getStandardLibraryPaths.result
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir = Utils.dirname(Utils.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt"
)
if Utils.isFile(origin_prefix_filename):
data_files.append(
(filename, "orig-prefix.txt")
)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(
filename = filename
)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort.
sys.exit(
SyntaxErrors.formatOutput(e)
)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(
main_module = main_module
)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(
getSourceDirectoryPath(main_module)
)
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(
0,
(binary_filename, None)
)
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
if Utils.getOS() == "NetBSD":
warning("Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported.")
copyUsedDLLs(
dist_dir = dist_dir,
standalone_entry_points = standalone_entry_points
)
for source_filename, target_filename in data_files:
shutil.copy2(
source_filename,
Utils.joinpath(
getStandaloneDirectoryPath(main_module),
target_filename
)
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(
(
"chmod",
"-x",
getResultFullpath(main_module)
)
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree = main_module,
clean_path = Options.shallClearPythonPathEnvironment()
)
else:
executeMain(
binary_filename = getResultFullpath(main_module),
clean_path = Options.shallClearPythonPathEnvironment()
)
def getStandardLibraryPaths():
""" Get the standard library paths.
"""
# Using the function object to cache its result, avoiding global variable
# usage.
if not hasattr(getStandardLibraryPaths, "result"):
os_filename = os.__file__
if os_filename.endswith(".pyc"):
os_filename = os_filename[:-1]
os_path = Utils.normcase(Utils.dirname(os_filename))
stdlib_paths = set([os_path])
# Happens for virtualenv situation, some modules will come from the link
# this points to.
if Utils.isLink(os_filename):
os_filename = Utils.readLink(os_filename)
stdlib_paths.add(Utils.normcase(Utils.dirname(os_filename)))
# Another possibility is "orig-prefix.txt" file near the os.py, which
# points to the original install.
orig_prefix_filename = Utils.joinpath(os_path, "orig-prefix.txt")
if Utils.isFile(orig_prefix_filename):
# Scan upwards, until we find a "bin" folder, with "activate" to
# locate the structural path to be added. We do not know for sure
# if there is a sub-directory under "lib" to use or not. So we try
# to detect it.
search = os_path
lib_part = ""
while os.path.splitdrive(search)[1] not in (os.path.sep, ""):
if Utils.isFile(Utils.joinpath(search,"bin/activate")) or \
Utils.isFile(Utils.joinpath(search,"scripts/activate")):
break
lib_part = Utils.joinpath(Utils.basename(search), lib_part)
search = Utils.dirname(search)
assert search and lib_part
stdlib_paths.add(
Utils.normcase(
Utils.joinpath(
open(orig_prefix_filename).read(),
lib_part,
)
)
)
# And yet another possibility, for MacOS Homebrew created virtualenv
# at least is a link ".Python", which points to the original install.
python_link_filename = Utils.joinpath(os_path, "..", ".Python")
if Utils.isLink(python_link_filename):
stdlib_paths.add(
Utils.normcase(
Utils.joinpath(
Utils.readLink(python_link_filename),
"lib"
)
)
)
getStandardLibraryPaths.result = [
Utils.normcase(stdlib_path)
for stdlib_path in
stdlib_paths
]
return getStandardLibraryPaths.result
def getSconsDataPath():
return Utils.dirname(__file__)
def getOutputFilename(self):
return Utils.dirname(self.getFilename())
def getSconsDataPath():
return Utils.dirname(__file__)
def getOutputFilename(self):
if self.main_added:
return Utils.dirname(self.getFilename())
else:
return CompiledPythonModule.getOutputFilename(self)
def _checkPluginPath(plugin_filename, module_package):
# Many branches, for the decision is very complex, pylint: disable=R0912
debug(
"Checking detail plug-in path '%s' '%s':",
plugin_filename,
module_package
)
plugin_info = considerFilename(
module_package = module_package,
module_filename = plugin_filename
)
if plugin_info is not None:
module, is_added = recurseTo(
module_filename = plugin_info[0],
module_relpath = plugin_info[1],
module_package = module_package,
module_kind = "py",
reason = "Lives in plug-in directory."
)
if module:
if not is_added:
warning(
"Recursed to %s '%s' at '%s' twice.",
"package" if module.isPythonPackage() else "module",
module.getName(),
plugin_info[0]
)
if not isSameModulePath(module.getFilename(), plugin_info[0]):
warning(
"Duplicate ignored '%s'.",
plugin_info[1]
)
return
debug(
"Recursed to %s %s %s",
module.getName(),
module.getPackage(),
module
)
if module.isPythonPackage():
package_filename = module.getFilename()
if Utils.isDir(package_filename):
# Must be a namespace package.
assert Utils.python_version >= 330
package_dir = package_filename
# Only include it, if it contains actual modules, which will
# recurse to this one and find it again.
useful = False
else:
package_dir = Utils.dirname(package_filename)
# Real packages will always be included.
useful = True
debug(
"Package directory %s",
package_dir
)
for sub_path, sub_filename in Utils.listDir(package_dir):
if sub_filename in ("__init__.py", "__pycache__"):
continue
assert sub_path != plugin_filename
if Importing.isPackageDir(sub_path) or \
sub_path.endswith(".py"):
_checkPluginPath(sub_path, module.getFullName())
else:
# Modules should always be included.
useful = True
if useful:
ModuleRegistry.addRootModule(module)
else:
warning("Failed to include module from '%s'.", plugin_info[0])
def getOutputFilename(self):
return Utils.dirname(self.getFilename())
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=R0912,R0914,R0915
# Print statements for stuff to show, the modules loaded.
if python_version >= 300:
command += '\nimport sys\nprint("\\n".join(sorted("import " + module.__name__ + " # sourcefile " + ' \
'module.__file__ for module in sys.modules.values() if hasattr(module, "__file__") and ' \
'module.__file__ != "<frozen>")), file = sys.stderr)' # do not read it
reduced_path = [
path_element
for path_element in
sys.path
if not Utils.areSamePaths(
path_element,
'.'
)
if not Utils.areSamePaths(
path_element,
Utils.dirname(sys.modules["__main__"].__file__)
)
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s;" % repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 300:
command = command.encode("ascii")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args = [sys.executable, "-s", "-S", "-v", tmp_filename],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
warning("There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
Tracing.printLine(line)
sys.exit("Error, please report the issue with above output.")
result = []
debug("Detecting imports:")
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 300:
module_name = module_name.decode("utf-8")
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
_detectedPrecompiledFile(
filename = filename,
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
_detectedSourceFile(
filename = filename,
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 300:
if module_name == "decimal":
module_name = "_decimal"
_detectedShlibFile(
filename = filename,
module_name = module_name
)
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
_detectedShlibFile(
filename = filename,
module_name = module_name
)
return result
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912,R0915
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=Utils.dirname(Utils.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt")
if Utils.isFile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort. If
# we are not in full compat, and user has not specified the Python
# versions he wants, tell him about the potential version problem.
error_message = SyntaxErrors.formatOutput(e)
if not Options.isFullCompat() and \
Options.getIntendedPythonVersion() is None:
if python_version < 300:
suggested_python_version_str = getSupportedPythonVersions()[-1]
else:
suggested_python_version_str = "2.7"
error_message += """
Nuitka is very syntax compatible with standard Python. It is currently running
with Python version '%s', you might want to specify more clearly with the use
of e.g. '--python-version=%s' option, if that's not the one expected.
""" % (python_version_str, suggested_python_version_str)
sys.exit(error_message)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(getSourceDirectoryPath(main_module))
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(0, (None, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = Utils.joinpath(
getStandaloneDirectoryPath(main_module), target_filename)
Utils.makePath(Utils.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment())
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment())
