def decideCompilationMode(is_top, module_name, for_pgo):
"""Decide the compilation mode for a module.
module_name - The module to decide compilation mode for.
for_pgo - consider PGO information or not
"""
result = Plugins.decideCompilation(module_name)
# Cannot change mode of __main__ to bytecode, that is not going
# to work currently.
if result == "bytecode" and is_top:
plugins_logger.warning("""\
Ignoring plugin decision to compile top level package '%s'
as bytecode, the extension module entry point is technically
required to compiled.""" % module_name)
result = "compiled"
# Include all of standard library as bytecode, for now. We need to identify
# which ones really need that.
if not is_top:
module_filename = Importing.locateModule(module_name=module_name,
parent_package=None,
level=0)[1]
if module_filename is not None and isStandardLibraryPath(
module_filename):
result = "bytecode"
# Plugins need to win over PGO, as they might know it better
if result is None and not for_pgo:
result = decideCompilationFromPGO(module_name=module_name)
# Default if neither plugins nor PGO have expressed an opinion
if result is None:
result = "compiled"
return result
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=R0912,R0914,R0915
# Print statements for stuff to show, the modules loaded.
if python_version >= 300:
command += '\nimport sys\nprint("\\n".join(sorted("import " + module.__name__ + " # sourcefile " + ' \
'module.__file__ for module in sys.modules.values() if hasattr(module, "__file__") and ' \
'module.__file__ != "<frozen>")), file = sys.stderr)' # do not read it
reduced_path = [
path_element
for path_element in
sys.path
if not Utils.areSamePaths(
path_element,
'.'
)
if not Utils.areSamePaths(
path_element,
Utils.dirname(sys.modules["__main__"].__file__)
)
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s;" % repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 300:
command = command.encode("ascii")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args = [sys.executable, "-s", "-S", "-v", tmp_filename],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
warning("There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
Tracing.printLine(line)
sys.exit("Error, please report the issue with above output.")
result = []
debug("Detecting imports:")
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 300:
module_name = module_name.decode("utf-8")
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
_detectedPrecompiledFile(
filename = filename,
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
_detectedSourceFile(
filename = filename,
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 300:
if module_name == "decimal":
module_name = "_decimal"
_detectedShlibFile(
filename = filename,
module_name = module_name
)
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
_detectedShlibFile(
filename = filename,
module_name = module_name
)
return result
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 0x300:
command = command.encode("utf8")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args=[sys.executable, "-s", "-S", "-v", tmp_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dict(os.environ, PYTHONIOENCODING="utf_8"),
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf-8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300:
if module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "shlib", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "shlib", filename))
for module_name, _prio, kind, filename in sorted(detections):
if kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "shlib":
_detectedShlibFile(filename=filename, module_name=module_name)
else:
assert False, kind
return result
def computeExpression(self, trace_collection):
# Many cases to deal with, pylint: disable=too-many-branches
# TODO: In fact, if the module is not a package, we don't have to insist
# on the "fromlist" that much, but normally it's not used for anything
# but packages, so it will be rare.
self._addUsedModules(trace_collection)
# Attempt to recurse if not already done.
if self.recurse_attempted:
if self.finding == "not-found":
# Importing and not finding, may raise an exception obviously.
trace_collection.onExceptionRaiseExit(BaseException)
else:
# If we know it exists, only RuntimeError shall occur.
trace_collection.onExceptionRaiseExit(RuntimeError)
# We stay here.
return self, None, None
module_name = self.subnode_name
if module_name.isCompileTimeConstant():
imported_module_name = module_name.getCompileTimeConstant()
if type(imported_module_name) in (str, unicode):
# TODO: This is not handling decoding errors all that well.
if str is not unicode and type(
imported_module_name) is unicode:
imported_module_name = str(imported_module_name)
self._attemptRecursion(trace_collection=trace_collection,
module_name=imported_module_name)
self.recurse_attempted = True
if self.finding == "absolute" and imported_module_name in hard_modules:
if isStandardLibraryPath(self.module_filename):
result = ExpressionImportModuleHard(
module_name=imported_module_name,
source_ref=self.source_ref)
return (
result,
"new_expression",
"Lowered import of standard library module %r to hard import."
% imported_module_name,
)
elif shallWarnUnusualCode():
unusual_logger.warning(
"%s Standard library module %r used from outside path %r."
% (
self.source_ref.getAsString(),
imported_module_name,
self.module_filename,
))
if self.finding == "built-in":
if imported_module_name in hard_modules:
result = ExpressionImportModuleHard(
module_name=imported_module_name,
source_ref=self.source_ref)
return (
result,
"new_expression",
"Lowered import of built-in module %r to hard import."
% imported_module_name,
)
self.type_shape = tshape_module_builtin
self.builtin_module = __import__(imported_module_name)
self._addUsedModules(trace_collection)
else:
# TODO: This doesn't preserve side effects.
# Non-strings is going to raise an error.
(
new_node,
change_tags,
message,
) = trace_collection.getCompileTimeComputationResult(
node=self,
computation=lambda: __import__(module_name.
getCompileTimeConstant()),
description=
"Replaced '__import__' call with non-string module name argument.",
)
# Must fail, must not go on when it doesn't.
assert change_tags == "new_raise", module_name
return new_node, change_tags, message
# Importing may raise an exception obviously, unless we know it will
# not.
if self.finding != "built-in":
trace_collection.onExceptionRaiseExit(BaseException)
# TODO: May return a module or module variable reference of some sort in
# the future with embedded modules.
return self, None, None
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=R0912,R0914,R0915
# Print statements for stuff to show, the modules loaded.
if python_version >= 300:
command += '\nimport sys\nprint("\\n".join(sorted("import " + module.__name__ + " # sourcefile " + ' \
'module.__file__ for module in sys.modules.values() if hasattr(module, "__file__") and ' \
'module.__file__ != "<frozen>")), file = sys.stderr)' # do not read it
reduced_path = [
path_element
for path_element in
sys.path
if not Utils.areSamePaths(
path_element,
'.'
)
if not Utils.areSamePaths(
path_element,
Utils.dirname(sys.modules["__main__"].__file__)
)
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s;" % repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 300:
command = command.encode("ascii")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args = [sys.executable, "-s", "-S", "-v", tmp_filename],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
warning("There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
Tracing.printLine(line)
sys.exit("Error, please report the issue with above output.")
result = []
debug("Detecting imports:")
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 300:
module_name = module_name.decode("utf-8")
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
_detectedPrecompiledFile(
filename = filename,
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
_detectedSourceFile(
filename = filename,
module_name = module_name,
result = result,
user_provided = user_provided,
technical = technical
)
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 300:
if module_name == "decimal":
module_name = "_decimal"
_detectedShlibFile(
filename = filename,
module_name = module_name
)
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
_detectedShlibFile(
filename = filename,
module_name = module_name
)
return result
def computeExpression(self, trace_collection):
# Attempt to recurse if not already done.
if self.follow_attempted:
if self.finding == "not-found":
# Importing and not finding, may raise an exception obviously.
trace_collection.onExceptionRaiseExit(BaseException)
else:
# If we know it exists, only RuntimeError shall occur.
trace_collection.onExceptionRaiseExit(RuntimeError)
# We stay here.
return self, None, None
module_name = self.subnode_name
if module_name.isCompileTimeConstant():
imported_module_name = module_name.getCompileTimeConstant()
module_filename = self._attemptFollow(
module_name=imported_module_name)
self.follow_attempted = True
if type(imported_module_name) in (str, unicode):
imported_module_name = resolveModuleName(imported_module_name)
if self.finding == "absolute" and imported_module_name in hard_modules:
if (isStandardLibraryPath(module_filename)
or imported_module_name == "pkg_resources"):
result = ExpressionImportModuleHard(
module_name=imported_module_name,
source_ref=self.source_ref)
return (
result,
"new_expression",
"Lowered import of standard library module '%s' to hard import."
% imported_module_name.asString(),
)
elif shallWarnUnusualCode():
unusual_logger.warning(
"%s: Standard library module '%s' used from outside path %r."
% (
self.source_ref.getAsString(),
imported_module_name.asString(),
self.module_filename,
))
if self.finding == "built-in":
if imported_module_name in hard_modules:
result = ExpressionImportModuleHard(
module_name=imported_module_name,
source_ref=self.source_ref)
return (
result,
"new_expression",
"Lowered import of built-in module '%s' to hard import."
% imported_module_name.asString(),
)
self.type_shape = tshape_module_builtin
self.builtin_module = __import__(imported_module_name)
else:
# TODO: This doesn't preserve side effects.
# Non-strings is going to raise an error.
(
new_node,
change_tags,
message,
) = trace_collection.getCompileTimeComputationResult(
node=self,
computation=lambda: __import__(module_name.
getCompileTimeConstant()),
description=
"Replaced '__import__' call with non-string module name argument.",
)
# Must fail, must not go on when it doesn't.
assert change_tags == "new_raise", module_name
return new_node, change_tags, message
# Importing may raise an exception obviously, unless we know it will
# not.
if self.finding != "built-in":
trace_collection.onExceptionRaiseExit(BaseException)
# TODO: May return a module or module variable reference of some sort in
# the future with embedded modules.
return self, None, None
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
if str is not bytes:
command = command.encode("utf8")
_stdout, stderr, exit_code = executeProcess(
command=(
sys.executable,
"-s",
"-S",
"-v",
"-c",
"import sys;exec(sys.stdin.read())",
),
stdin=command,
env=dict(os.environ, PYTHONIOENCODING="utf-8"),
)
assert type(stderr) is bytes
# Don't let errors here go unnoticed.
if exit_code != 0:
# An error by the user pressing CTRL-C should not lead to the below output.
if b"KeyboardInterrupt" in stderr:
general.sysexit("Pressed CTRL-C while detecting early imports.")
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"from" and python_version < 0x300:
filename = parts[1][len(b"from "):]
if str is not bytes: # For consistency, and maybe later reuse
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
else:
assert False
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300 and module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "extension", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "extension", filename))
for module_name, _priority, kind, filename in sorted(detections):
if isStandardLibraryNoAutoInclusionModule(module_name):
continue
if kind == "extension":
_detectedExtensionModule(
filename=filename,
module_name=module_name,
result=result,
technical=technical,
)
elif kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
else:
assert False, kind
return result
