def _generateExpressionCode(to_name, expression, emit, context, allow_none=False):
# This is a dispatching function for every expression.
if expression is None and allow_none:
return None
# Make sure we don't generate code twice for any node, this uncovers bugs
# where nodes are shared in the tree, which is not allowed.
assert not hasattr(expression, "code_generated"), expression
expression.code_generated = True
if not expression.isExpression():
printError("No expression %r" % expression)
expression.dump()
assert False, expression
try:
code_generator = expression_dispatch_dict[expression.kind]
except KeyError:
raise NuitkaNodeDesignError(
expression.__class__.__name__,
"Need to provide code generation as well",
expression.kind,
)
with context.withCurrentSourceCodeReference(expression.getSourceReference()):
code_generator(
to_name=to_name, expression=expression, emit=emit, context=context
)
def _generateExpressionCode(to_name,
expression,
emit,
context,
allow_none=False):
# This is a dispatching function for every expression.
if expression is None and allow_none:
return None
# Make sure we don't generate code twice for any node, this uncovers bugs
# where nodes are shared in the tree, which is not allowed.
assert not hasattr(expression, "code_generated"), expression
expression.code_generated = True
old_source_ref = context.setCurrentSourceCodeReference(
expression.getSourceReference())
if not expression.isExpression():
printError("No expression %r" % expression)
expression.dump()
assert False, expression
expression_dispatch_dict[expression.kind](to_name=to_name,
expression=expression,
emit=emit,
context=context)
context.setCurrentSourceCodeReference(old_source_ref)
def _generateExpressionCode(to_name, expression, emit, context, allow_none = False):
# This is a dispatching function for every expression.
if expression is None and allow_none:
return None
# Make sure we don't generate code twice for any node, this uncovers bugs
# where nodes are shared in the tree, which is not allowed.
assert not hasattr(expression, "code_generated"), expression
expression.code_generated = True
old_source_ref = context.setCurrentSourceCodeReference(expression.getSourceReference())
if not expression.isExpression():
printError("No expression %r" % expression)
expression.dump()
assert False, expression
expression_dispatch_dict[expression.kind](
to_name = to_name,
expression = expression,
emit = emit,
context = context
)
context.setCurrentSourceCodeReference(old_source_ref)
def generateStatementCode(statement, emit, context):
try:
statement_dispatch_dict[statement.kind](statement=statement,
emit=emit,
context=context)
# Complain if any temporary was not dealt with yet.
assert not context.getCleanupTempnames(), context.getCleanupTempnames()
except Exception:
printError("Problem with %r at %s" %
(statement, statement.getSourceReference().getAsString()))
raise
def generateStatementCode(statement, emit, context):
try:
statement_dispatch_dict[statement.kind](
statement=statement, emit=emit, context=context
)
# Complain if any temporary was not dealt with yet.
assert not context.getCleanupTempnames(), context.getCleanupTempnames()
except Exception:
printError(
"Problem with %r at %s"
% (statement, statement.getSourceReference().getAsString())
)
raise
def generateExpressionCode(to_name,
expression,
emit,
context,
allow_none=False):
try:
_generateExpressionCode(to_name=to_name,
expression=expression,
emit=emit,
context=context,
allow_none=allow_none)
except Exception:
printError("Problem with %r at %s" %
(expression, "" if expression is None else
expression.getSourceReference().getAsString()))
raise
def generateExpressionCode(to_name, expression, emit, context,
allow_none = False):
try:
_generateExpressionCode(
to_name = to_name,
expression = expression,
emit = emit,
context = context,
allow_none = allow_none
)
except Exception:
printError(
"Problem with %r at %s" % (
expression,
""
if expression is None else
expression.getSourceReference().getAsString()
)
)
raise
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
import tempfile
tmp_file, tmp_filename = tempfile.mkstemp()
try:
if python_version >= 0x300:
command = command.encode("utf8")
os.write(tmp_file, command)
os.close(tmp_file)
process = subprocess.Popen(
args=[sys.executable, "-s", "-S", "-v", tmp_filename],
stdin=getNullInput(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dict(os.environ, PYTHONIOENCODING="utf_8"),
)
_stdout, stderr = process.communicate()
finally:
os.unlink(tmp_filename)
# Don't let errors here go unnoticed.
if process.returncode != 0:
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
# print(line)
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf-8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300:
if module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "shlib", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf-8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "shlib", filename))
for module_name, _prio, kind, filename in sorted(detections):
if kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "shlib":
_detectedShlibFile(filename=filename, module_name=module_name)
else:
assert False, kind
return result
def _detectImports(command, user_provided, technical):
# This is pretty complicated stuff, with variants to deal with.
# pylint: disable=too-many-branches,too-many-locals,too-many-statements
# Print statements for stuff to show, the modules loaded.
if python_version >= 0x300:
command += """
print("\\n".join(sorted(
"import %s # sourcefile %s" % (module.__name__, module.__file__)
for module in sys.modules.values()
if getattr(module, "__file__", None) not in (None, "<frozen>"
))), file = sys.stderr)"""
reduced_path = [
path_element for path_element in sys.path
if not areSamePaths(path_element, ".") if not areSamePaths(
path_element, os.path.dirname(sys.modules["__main__"].__file__))
]
# Make sure the right import path (the one Nuitka binary is running with)
# is used.
command = ("import sys; sys.path = %s; sys.real_prefix = sys.prefix;" %
repr(reduced_path)) + command
if str is not bytes:
command = command.encode("utf8")
_stdout, stderr, exit_code = executeProcess(
command=(
sys.executable,
"-s",
"-S",
"-v",
"-c",
"import sys;exec(sys.stdin.read())",
),
stdin=command,
env=dict(os.environ, PYTHONIOENCODING="utf-8"),
)
assert type(stderr) is bytes
# Don't let errors here go unnoticed.
if exit_code != 0:
# An error by the user pressing CTRL-C should not lead to the below output.
if b"KeyboardInterrupt" in stderr:
general.sysexit("Pressed CTRL-C while detecting early imports.")
general.warning(
"There is a problem with detecting imports, CPython said:")
for line in stderr.split(b"\n"):
printError(line)
general.sysexit("Error, please report the issue with above output.")
result = []
detections = []
for line in stderr.replace(b"\r", b"").split(b"\n"):
if line.startswith(b"import "):
parts = line.split(b" # ", 2)
module_name = parts[0].split(b" ", 2)[1]
origin = parts[1].split()[0]
if python_version >= 0x300:
module_name = module_name.decode("utf8")
module_name = ModuleName(module_name)
if origin == b"precompiled":
# This is a ".pyc" file that was imported, even before we have a
# chance to do anything, we need to preserve it.
filename = parts[1][len(b"precompiled from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 3, "precompiled", filename))
elif origin == b"from" and python_version < 0x300:
filename = parts[1][len(b"from "):]
if str is not bytes: # For consistency, and maybe later reuse
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
else:
assert False
elif origin == b"sourcefile":
filename = parts[1][len(b"sourcefile "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
if filename.endswith(".py"):
detections.append((module_name, 2, "sourcefile", filename))
elif filename.endswith(".pyc"):
detections.append(
(module_name, 3, "precompiled", filename))
elif not filename.endswith("<frozen>"):
# Python3 started lying in "__name__" for the "_decimal"
# calls itself "decimal", which then is wrong and also
# clashes with "decimal" proper
if python_version >= 0x300 and module_name == "decimal":
module_name = ModuleName("_decimal")
detections.append((module_name, 2, "extension", filename))
elif origin == b"dynamically":
# Shared library in early load, happens on RPM based systems and
# or self compiled Python installations.
filename = parts[1][len(b"dynamically loaded from "):]
if python_version >= 0x300:
filename = filename.decode("utf8")
# Do not leave standard library when freezing.
if not isStandardLibraryPath(filename):
continue
detections.append((module_name, 1, "extension", filename))
for module_name, _priority, kind, filename in sorted(detections):
if isStandardLibraryNoAutoInclusionModule(module_name):
continue
if kind == "extension":
_detectedExtensionModule(
filename=filename,
module_name=module_name,
result=result,
technical=technical,
)
elif kind == "precompiled":
_detectedPrecompiledFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
elif kind == "sourcefile":
_detectedSourceFile(
filename=filename,
module_name=module_name,
result=result,
user_provided=user_provided,
technical=technical,
)
else:
assert False, kind
return result
