def _attachVariable(node, provider):
# print "Late reference", node.getVariableName(), "for", provider, "caused at", node, "of", node.getParent()
variable_name = node.getVariableName()
was_taken = provider.hasTakenVariable(variable_name)
variable = provider.getVariableForReference(
variable_name=variable_name)
node.setVariable(variable)
# Need to catch functions with "exec" and closure variables not allowed.
if python_version < 300 and \
not was_taken and \
provider.isExpressionFunctionBodyBase() and \
variable.getOwner() is not provider:
parent_provider = provider.getParentVariableProvider()
while parent_provider.isExpressionClassBody():
parent_provider = parent_provider.getParentVariableProvider()
if parent_provider.isExpressionFunctionBody() and \
parent_provider.isUnqualifiedExec():
SyntaxErrors.raiseSyntaxError(
reason = PythonVersions.\
getErrorMessageExecWithNestedFunction() % \
parent_provider.getName(),
source_ref = parent_provider.getExecSourceRef(),
col_offset = None,
display_file = True,
display_line = True,
source_line = None
)
def _handleFutureImport(provider, node, source_ref):
# Don't allow future imports in functions or classes.
if not provider.isCompiledPythonModule():
SyntaxErrors.raiseSyntaxError(
reason = """\
from __future__ imports must occur at the beginning of the file""",
col_offset = 8
if python_version >= 300 or \
not Options.isFullCompat()
else None,
source_ref = source_ref
)
for import_desc in node.names:
object_name, _local_name = import_desc.name, import_desc.asname
_enableFutureFeature(
object_name = object_name,
future_spec = source_ref.getFutureSpec(),
source_ref = source_ref
)
# Remember it for checks to be applied once module is complete, e.g. if
# they are all at module start.
node.source_ref = source_ref
_future_import_nodes.append(node)
def onLeaveNode(self, node):
# Return statements in generators are not really that, instead they are
# exception raises, fix that up now. Doing it right from the onset,
# would be a bit more difficult, as the knowledge that something is a
# generator, requires a second pass.
if node.isStatementReturn():
return_consumer = node.getParentReturnConsumer()
if return_consumer.isExpressionFunctionBody() and \
return_consumer.isGenerator():
return_value = node.getExpression()
if python_version < 330:
if not return_value.isExpressionReturnedValueRef() and \
(not return_value.isExpressionConstantRef() or \
return_value.getConstant() is not None):
SyntaxErrors.raiseSyntaxError(
"'return' with argument inside generator",
source_ref=node.getSourceReference(),
)
node.replaceWith(
StatementGeneratorReturn(
expression=return_value,
source_ref=node.getSourceReference()))
def _enableFutureFeature(object_name, future_spec, source_ref):
if object_name == "unicode_literals":
future_spec.enableUnicodeLiterals()
elif object_name == "absolute_import":
future_spec.enableAbsoluteImport()
elif object_name == "division":
future_spec.enableFutureDivision()
elif object_name == "print_function":
future_spec.enableFuturePrint()
elif object_name == "barry_as_FLUFL" and python_version >= 300:
future_spec.enableBarry()
elif object_name == "generator_stop":
future_spec.enableGeneratorStop()
elif object_name == "braces":
SyntaxErrors.raiseSyntaxError(
"not a chance",
source_ref
)
elif object_name in ("nested_scopes", "generators", "with_statement"):
# These are enabled in all cases already.
pass
else:
SyntaxErrors.raiseSyntaxError(
"future feature %s is not defined" % object_name,
source_ref
)
def _handleNonLocal(node):
# Take closure variables for non-local declarations.
for non_local_names, source_ref in node.getNonlocalDeclarations():
for non_local_name in non_local_names:
variable = node.getClosureVariable(
variable_name = non_local_name
)
if variable.isModuleVariable():
SyntaxErrors.raiseSyntaxError(
"no binding for nonlocal '%s' found" % (
non_local_name
),
source_ref = None
if isFullCompat() and \
python_version < 340 else
source_ref,
display_file = not isFullCompat() or \
python_version >= 340,
display_line = not isFullCompat() or \
python_version >= 340
)
node.registerProvidedVariable(variable)
addVariableUsage(variable, node)
def _enableFutureFeature(node, object_name, future_spec, source_ref):
if object_name == "unicode_literals":
future_spec.enableUnicodeLiterals()
elif object_name == "absolute_import":
future_spec.enableAbsoluteImport()
elif object_name == "division":
future_spec.enableFutureDivision()
elif object_name == "print_function":
future_spec.enableFuturePrint()
elif object_name == "barry_as_FLUFL" and python_version >= 300:
future_spec.enableBarry()
elif object_name == "generator_stop":
future_spec.enableGeneratorStop()
elif object_name == "braces":
SyntaxErrors.raiseSyntaxError(
"not a chance",
source_ref.atColumnNumber(node.col_offset)
)
elif object_name in ("nested_scopes", "generators", "with_statement"):
# These are enabled in all cases already.
pass
else:
SyntaxErrors.raiseSyntaxError(
"future feature %s is not defined" % object_name,
source_ref.atColumnNumber(node.col_offset)
)
def buildStatementContinueLoop(node, source_ref):
if getBuildContext() == "finally":
if not Options.isFullCompat() or Utils.python_version >= 300:
col_offset = node.col_offset - 9
else:
col_offset = None
if Utils.python_version >= 300 and Options.isFullCompat():
source_line = ""
else:
source_line = None
SyntaxErrors.raiseSyntaxError(
"'continue' not supported inside 'finally' clause",
source_ref,
col_offset = col_offset,
source_line = source_line
)
statements = makeTryFinallyIndicatorStatements(
is_loop_exit = True,
source_ref = source_ref
)
statements.append(
StatementContinueLoop(
source_ref = source_ref
)
)
return makeStatementsSequenceOrStatement(
statements = statements,
source_ref = source_ref
)
def _markAsGenerator(provider, node, source_ref):
if provider.isPythonModule():
SyntaxErrors.raiseSyntaxError(
"'yield' outside function", source_ref,
None if Utils.python_version < 300 else node.col_offset)
provider.markAsGenerator()
def buildParameterSpec(provider, name, node, source_ref):
kind = getKind(node)
assert kind in ("FunctionDef", "Lambda",
"AsyncFunctionDef"), "unsupported for kind " + kind
def extractArg(arg):
if arg is None:
return None
elif type(arg) is str:
return mangleName(arg, provider)
elif getKind(arg) == "Name":
return mangleName(arg.id, provider)
elif getKind(arg) == "arg":
return mangleName(arg.arg, provider)
elif getKind(arg) == "Tuple":
return tuple(extractArg(arg) for arg in arg.elts)
else:
assert False, getKind(arg)
result = ParameterSpec(
name=name,
normal_args=[extractArg(arg) for arg in node.args.args],
kw_only_args=[extractArg(arg) for arg in node.args.kwonlyargs]
if Utils.python_version >= 300 else [],
list_star_arg=extractArg(node.args.vararg),
dict_star_arg=extractArg(node.args.kwarg),
default_count=len(node.args.defaults))
message = result.checkValid()
if message is not None:
SyntaxErrors.raiseSyntaxError(message, source_ref)
return result
def buildReturnNode(provider, node, source_ref):
if not provider.isExpressionFunctionBody() or \
provider.isClassDictCreation():
SyntaxErrors.raiseSyntaxError(
"'return' outside function",
source_ref,
None if Utils.python_version < 300 else (
node.col_offset
if provider.isPythonModule() else
node.col_offset+4
)
)
expression = buildNode(provider, node.value, source_ref, allow_none = True)
if expression is None:
expression = ExpressionConstantRef(
constant = None,
source_ref = source_ref,
user_provided = True
)
return StatementReturn(
expression = expression,
source_ref = source_ref
)
def _handleNonLocal(node):
# Take closure variables for non-local declarations.
for non_local_names, source_ref in node.getNonlocalDeclarations():
for non_local_name in non_local_names:
variable = node.getClosureVariable(
variable_name=non_local_name)
if variable.isModuleVariable():
SyntaxErrors.raiseSyntaxError(
"no binding for nonlocal '%s' found" % (
non_local_name
),
source_ref = None
if isFullCompat() and \
python_version < 340 else
source_ref,
display_file = not isFullCompat() or \
python_version >= 340,
display_line = not isFullCompat() or \
python_version >= 340
)
node.registerProvidedVariable(variable)
variable.addVariableUser(node)
def onLeaveNode(self, node):
# Return statements in generators are not really that, instead they are
# exception raises, fix that up now. Doing it right from the onset,
# would be a bit more difficult, as the knowledge that something is a
# generator, requires a second pass.
if node.isStatementReturn():
return_consumer = node.getParentReturnConsumer()
if return_consumer.isExpressionFunctionBody() and \
return_consumer.isGenerator():
return_value = node.getExpression()
if python_version < 330:
if not return_value.isExpressionReturnedValueRef() and \
(not return_value.isExpressionConstantRef() or \
return_value.getConstant() is not None):
SyntaxErrors.raiseSyntaxError(
"'return' with argument inside generator",
source_ref = node.getSourceReference(),
)
node.replaceWith(
StatementGeneratorReturn(
expression = return_value,
source_ref = node.getSourceReference()
)
)
def _markAsGenerator(provider, node, source_ref):
if provider.isPythonModule():
SyntaxErrors.raiseSyntaxError(
"'yield' outside function",
source_ref,
None if Utils.python_version < 300 else node.col_offset
)
provider.markAsGenerator()
def onEnterNode(self, node):
if python_version < 300 and node.isStatementDelVariable():
variable = node.getTargetVariableRef().getVariable()
if not variable.isModuleVariable() and \
variable.isSharedAmongScopes():
SyntaxErrors.raiseSyntaxError(
"""\
can not delete variable '%s' referenced in nested scope""" %
(variable.getName()), node.getSourceReference())
elif node.isStatementsFrame():
node.updateLocalNames()
def _checkInsideGenerator(provider, node, source_ref):
if provider.isCompiledPythonModule():
SyntaxErrors.raiseSyntaxError(
"'yield' outside function", source_ref,
None if python_version < 300 else node.col_offset)
if provider.isExpressionCoroutineObjectBody():
SyntaxErrors.raiseSyntaxError(
"'%s' inside async function" %
("yield" if node.__class__ is ast.Yield else "yield from", ),
source_ref, node.col_offset + 3)
assert provider.isExpressionGeneratorObjectBody(), provider
def buildParameterSpec(provider, name, node, source_ref):
kind = getKind(node)
assert kind in ("FunctionDef", "Lambda"), "unsupported for kind " + kind
def extractArg(arg):
if arg is None:
return None
elif type(arg) is str:
return mangleName(arg, provider)
elif getKind(arg) == "Name":
return mangleName(arg.id, provider)
elif getKind(arg) == "arg":
return mangleName(arg.arg, provider)
elif getKind(arg) == "Tuple":
return tuple(
extractArg(arg)
for arg in
arg.elts
)
else:
assert False, getKind(arg)
result = ParameterSpec(
name = name,
normal_args = [
extractArg(arg)
for arg in
node.args.args
],
kw_only_args = [
extractArg(arg)
for arg in
node.args.kwonlyargs
]
if Utils.python_version >= 300 else
[],
list_star_arg = extractArg(node.args.vararg),
dict_star_arg = extractArg(node.args.kwarg),
default_count = len(node.args.defaults)
)
message = result.checkValid()
if message is not None:
SyntaxErrors.raiseSyntaxError(
message,
source_ref
)
return result
def checkFutureImportsOnlyAtStart(body):
# Check if a __future__ imports really were at the beginning of the file.
for node in body:
if node in _future_import_nodes:
_future_import_nodes.remove(node)
else:
if _future_import_nodes:
SyntaxErrors.raiseSyntaxError(
"""\
from __future__ imports must occur at the beginning of the file""",
_future_import_nodes[0].source_ref.atColumnNumber(
_future_import_nodes[0].col_offset
)
)
def _handleNonLocal(node):
# Take closure variables for non-local declarations.
for non_local_names, source_ref in node.getNonlocalDeclarations():
for non_local_name in non_local_names:
variable = node.getClosureVariable(
variable_name=non_local_name)
if variable.isModuleVariable():
SyntaxErrors.raiseSyntaxError(
"no binding for nonlocal '%s' found" %
(non_local_name), source_ref)
node.registerProvidedVariable(variable)
variable.addVariableUser(node)
def onEnterNode(self, node):
if python_version < 300 and node.isStatementDelVariable():
variable = node.getTargetVariableRef().getVariable()
if not variable.isModuleVariable() and \
variable.isSharedAmongScopes():
SyntaxErrors.raiseSyntaxError(
reason="""\
can not delete variable '%s' referenced in nested scope""" %
(variable.getName()),
source_ref=(None if isFullCompat() else
node.getSourceReference()),
display_file=not isFullCompat(),
display_line=not isFullCompat())
elif node.isStatementsFrame():
node.updateLocalNames()
def checkFutureImportsOnlyAtStart(body):
# Check if a __future__ imports really were at the beginning of the file.
for node in body:
if node in _future_import_nodes:
_future_import_nodes.remove(node)
else:
if _future_import_nodes:
SyntaxErrors.raiseSyntaxError(
reason = """\
from __future__ imports must occur at the beginning of the file""",
col_offset = 1
if Utils.python_version >= 300 or \
not Options.isFullCompat() else
None,
source_ref = _future_import_nodes[0].source_ref
)
def checkFutureImportsOnlyAtStart(body):
# Check if a __future__ imports really were at the beginning of the file.
for node in body:
if node in _future_import_nodes:
_future_import_nodes.remove(node)
else:
if _future_import_nodes:
SyntaxErrors.raiseSyntaxError(
reason = """\
from __future__ imports must occur at the beginning of the file""",
col_offset = 1
if python_version >= 300 or \
not Options.isFullCompat() else
None,
source_ref = _future_import_nodes[0].source_ref
)
def _checkInsideGenerator(provider, node, source_ref):
if provider.isCompiledPythonModule():
SyntaxErrors.raiseSyntaxError(
"'yield' outside function",
source_ref,
None if python_version < 300 else node.col_offset
)
if provider.isExpressionCoroutineObjectBody():
SyntaxErrors.raiseSyntaxError(
"'%s' inside async function" % (
"yield" if node.__class__ is ast.Yield else "yield from",
),
source_ref,
node.col_offset+3
)
assert provider.isExpressionGeneratorObjectBody(), provider
def _checkInsideGenerator(provider, node, source_ref):
if provider.isCompiledPythonModule():
SyntaxErrors.raiseSyntaxError(
"'yield' outside function",
source_ref.atColumnNumber(node.col_offset)
)
# This is forbidden in 3.5, but allows in 3.6
if provider.isExpressionCoroutineObjectBody() and python_version < 360:
SyntaxErrors.raiseSyntaxError(
"'%s' inside async function" % (
"yield" if node.__class__ is ast.Yield else "yield from",
),
source_ref.atColumnNumber(node.col_offset)
)
assert provider.isExpressionGeneratorObjectBody() or \
provider.isExpressionCoroutineObjectBody(), provider
def _readSourceCodeFromFilename2(source_filename):
# Detect the encoding.
with open(source_filename, "rU") as source_file:
encoding = _detectEncoding2(source_file)
source_code = source_file.read()
# Try and detect SyntaxError from missing or wrong encodings.
if type(source_code) is not unicode and encoding == "ascii":
try:
_source_code = source_code.decode(encoding)
except UnicodeDecodeError as e:
lines = source_code.split('\n')
so_far = 0
for count, line in enumerate(lines):
so_far += len(line) + 1
if so_far > e.args[2]:
break
else:
# Cannot happen, decode error implies non-empty.
count = -1
wrong_byte = re.search(
"byte 0x([a-f0-9]{2}) in position",
str(e)
).group(1)
SyntaxErrors.raiseSyntaxError(
reason = """\
Non-ASCII character '\\x%s' in file %s on line %d, but no encoding declared; \
see http://python.org/dev/peps/pep-0263/ for details""" % (
wrong_byte,
source_filename,
count+1,
),
source_ref = SourceCodeReferences.fromFilename(
source_filename
).atLineNumber(count+1),
display_line = False
)
return source_code
def _readSourceCodeFromFilename2(source_filename):
# Detect the encoding.
with open(source_filename, "rU") as source_file:
encoding = _detectEncoding2(source_file)
source_code = source_file.read()
# Try and detect SyntaxError from missing or wrong encodings.
if type(source_code) is not unicode and encoding == "ascii":
try:
_source_code = source_code.decode(encoding)
except UnicodeDecodeError as e:
lines = source_code.split('\n')
so_far = 0
for count, line in enumerate(lines):
so_far += len(line) + 1
if so_far > e.args[2]:
break
else:
# Cannot happen, decode error implies non-empty.
count = -1
wrong_byte = re.search(
"byte 0x([a-f0-9]{2}) in position",
str(e)
).group(1)
SyntaxErrors.raiseSyntaxError(
reason = """\
Non-ASCII character '\\x%s' in file %s on line %d, but no encoding declared; \
see http://python.org/dev/peps/pep-0263/ for details""" % (
wrong_byte,
source_filename,
count+1,
),
source_ref = SourceCodeReferences.fromFilename(
source_filename
).atLineNumber(count+1),
display_line = False
)
return source_code
def onEnterNode(self, node):
if python_version < 300 and node.isStatementDelVariable():
variable = node.getTargetVariableRef().getVariable()
if not variable.isModuleVariable() and \
isSharedAmongScopes(variable):
SyntaxErrors.raiseSyntaxError(
reason = """\
can not delete variable '%s' referenced in nested scope""" % (
variable.getName()
),
source_ref = (
None if isFullCompat() else node.getSourceReference()
),
display_file = not isFullCompat(),
display_line = not isFullCompat()
)
elif node.isStatementsFrame():
node.updateLocalNames()
def _attachVariable(node, provider):
# print "Late reference", node.getVariableName(), "for", provider, "caused at", node, "of", node.getParent()
variable_name = node.getVariableName()
was_taken = provider.hasTakenVariable(variable_name)
variable = provider.getVariableForReference(
variable_name = variable_name
)
node.setVariable(
variable
)
# Need to catch functions with "exec" and closure variables not allowed.
if python_version < 300 and \
not was_taken and \
provider.isExpressionFunctionBody() and \
variable.getOwner() is not provider:
parent_provider = provider.getParentVariableProvider()
while parent_provider.isExpressionFunctionBody() and \
parent_provider.isClassDictCreation():
parent_provider = parent_provider.getParentVariableProvider()
if parent_provider.isExpressionFunctionBody() and \
parent_provider.isUnqualifiedExec():
SyntaxErrors.raiseSyntaxError(
reason = PythonVersions.\
getErrorMessageExecWithNestedFunction() % \
parent_provider.getName(),
source_ref = parent_provider.getExecSourceRef(),
col_offset = None,
display_file = True,
display_line = True,
source_line = None
)
def handleNonlocalDeclarationNode(provider, node, source_ref):
# Need to catch the error of declaring a parameter variable as global
# ourselves here. The AST parsing doesn't catch it, but we can do it here.
parameters = provider.getParameters()
for variable_name in node.names:
if variable_name in parameters.getParameterNames():
SyntaxErrors.raiseSyntaxError(
reason = "name '%s' is parameter and nonlocal" % (
variable_name
),
source_ref = None
if Options.isFullCompat() and \
Utils.python_version < 340 else
source_ref,
display_file = not Options.isFullCompat() or \
Utils.python_version >= 340,
display_line = not Options.isFullCompat() or \
Utils.python_version >= 340
)
provider.addNonlocalsDeclaration(node.names, source_ref)
return None
def _handleFutureImport(provider, node, source_ref):
# Don't allow future imports in functions or classes.
if not provider.isCompiledPythonModule():
SyntaxErrors.raiseSyntaxError(
reason = """\
from __future__ imports must occur at the beginning of the file""",
col_offset = 8
if python_version >= 300 or \
not Options.isFullCompat()
else None,
source_ref = source_ref
)
for import_desc in node.names:
object_name, _local_name = import_desc.name, import_desc.asname
_enableFutureFeature(object_name=object_name,
future_spec=source_ref.getFutureSpec(),
source_ref=source_ref)
# Remember it for checks to be applied once module is complete, e.g. if
# they are all at module start.
node.source_ref = source_ref
_future_import_nodes.append(node)
def buildStatementContinueLoop(node, source_ref):
# Python forbids this, although technically it's probably not much of
# an issue.
if getBuildContext() == "finally":
if not Options.isFullCompat() or Utils.python_version >= 300:
col_offset = node.col_offset - 9
else:
col_offset = None
if Utils.python_version >= 300 and Options.isFullCompat():
source_line = ""
else:
source_line = None
SyntaxErrors.raiseSyntaxError(
"'continue' not supported inside 'finally' clause",
source_ref,
col_offset = col_offset,
source_line = source_line
)
return StatementContinueLoop(
source_ref = source_ref
)
def handleSyntaxError(e):
# Syntax or indentation errors, output them to the user and abort. If
# we are not in full compat, and user has not specified the Python
# versions he wants, tell him about the potential version problem.
error_message = SyntaxErrors.formatOutput(e)
if not Options.isFullCompat():
if python_version < 300:
suggested_python_version_str = getSupportedPythonVersions()[-1]
else:
suggested_python_version_str = "2.7"
error_message += """
Nuitka is very syntax compatible with standard Python. It is currently running
with Python version '%s', you might want to specify more clearly with the use
of the precise Python interpreter binary and '-m nuitka', e.g. use this
'python%s -m nuitka' option, if that's not the one the program expects.
""" % (python_version_str, suggested_python_version_str)
sys.exit(error_message)
def handleSyntaxError(e):
# Syntax or indentation errors, output them to the user and abort. If
# we are not in full compat, and user has not specified the Python
# versions he wants, tell him about the potential version problem.
error_message = SyntaxErrors.formatOutput(e)
if not Options.isFullCompat():
if python_version < 300:
suggested_python_version_str = getSupportedPythonVersions()[-1]
else:
suggested_python_version_str = "2.7"
error_message += """
Nuitka is very syntax compatible with standard Python. It is currently running
with Python version '%s', you might want to specify more clearly with the use
of the precise Python interpreter binary and '-m nuitka', e.g. use this
'python%s -m nuitka' option, if that's not the one the program expects.
""" % (
python_version_str,
suggested_python_version_str,
)
sys.exit(error_message)
def buildImportFromNode(provider, node, source_ref):
# "from .. import .." statements. This may trigger a star import, or
# multiple names being looked up from the given module variable name.
# This is pretty complex, pylint: disable=R0912,R0914
module_name = node.module if node.module is not None else ""
level = node.level
# Importing from "__future__" module may enable flags to the parser,
# that we need to know about, handle that.
if module_name == "__future__":
_handleFutureImport(provider, node, source_ref)
target_names = []
import_names = []
# Mapping imported "fromlist" to assigned "fromlist" if any, handling the
# star case as well.
for import_desc in node.names:
object_name, local_name = import_desc.name, import_desc.asname
if object_name == '*':
target_names.append(None)
assert local_name is None
else:
target_names.append(
local_name if local_name is not None else object_name)
import_names.append(object_name)
# Star imports get special treatment.
if None in target_names:
# More than "*" is a syntax error in Python, need not care about this at
# all, it's only allowed value for import list in this case.
assert target_names == [None]
# Python3 made it so that these can only occur on the module level,
# so this a syntax error if not there. For Python2 it is OK to
# occur everywhere though.
if not provider.isCompiledPythonModule() and python_version >= 300:
SyntaxErrors.raiseSyntaxError(
"import * only allowed at module level",
provider.getSourceReference())
# Functions with star imports get a marker.
if provider.isExpressionFunctionBody():
provider.markAsStarImportContaining()
return StatementImportStar(module_import=ExpressionImportModule(
module_name=module_name,
import_list=('*', ),
level=level,
source_ref=source_ref),
source_ref=source_ref)
else:
def makeImportName(import_name):
if module_name == "__future__":
# Make "__future__" imports tie hard immediately, they cannot be
# any other way.
return ExpressionImportModuleHard(module_name="__future__",
import_name=import_name,
source_ref=source_ref)
else:
# Refer to be module, or a clone of the reference if need be.
return ExpressionImportName(module=imported_from_module,
import_name=import_name,
source_ref=source_ref)
imported_from_module = ExpressionImportModule(
module_name=module_name,
import_list=tuple(import_names),
level=level,
source_ref=source_ref)
# If we have multiple names to import, consider each.
multi_names = len(target_names) > 1
statements = []
if multi_names:
tmp_import_from = provider.allocateTempVariable(
temp_scope=provider.allocateTempScope("import_from"),
name="module")
statements.append(
StatementAssignmentVariable(
variable_ref=ExpressionTargetTempVariableRef(
variable=tmp_import_from, source_ref=source_ref),
source=imported_from_module,
source_ref=source_ref))
imported_from_module = ExpressionTempVariableRef(
variable=tmp_import_from, source_ref=source_ref)
import_statements = []
first = True
for target_name, import_name in zip(target_names, import_names):
# Make a clone of the variable reference, if we are going to use
# another one.
if not first:
imported_from_module = imported_from_module.makeClone()
first = False
import_statements.append(
StatementAssignmentVariable(
variable_ref=ExpressionTargetVariableRef(
variable_name=mangleName(target_name, provider),
source_ref=source_ref),
source=makeImportName(import_name=import_name, ),
source_ref=source_ref))
# Release the temporary module value as well.
if multi_names:
statements.append(
makeTryFinallyStatement(provider=provider,
tried=import_statements,
final=(StatementReleaseVariable(
variable=tmp_import_from,
source_ref=source_ref), ),
source_ref=source_ref))
else:
statements.extend(import_statements)
# Note: Each import is sequential. It can succeed, and the failure of a
# later one is not undoing previous ones. We can therefore have a
# sequence of imports that each only import one thing therefore.
return StatementsSequence(statements=mergeStatements(statements),
source_ref=source_ref)
def handleGlobalDeclarationNode(provider, node, source_ref):
# On the module level, there is nothing to do. TODO: Probably a warning
# would be warranted.
if provider.isPythonModule():
return None
# Need to catch the error of declaring a parameter variable as global
# ourselves here. The AST parsing doesn't catch it, so we check here.
try:
parameters = provider.getParameters()
for variable_name in node.names:
if variable_name in parameters.getParameterNames():
SyntaxErrors.raiseSyntaxError(
reason = "name '%s' is %s and global" % (
variable_name,
"local"
if Utils.python_version < 300 else
"parameter"
),
source_ref = (
source_ref
if not Options.isFullCompat() or \
Utils.python_version >= 340 else
provider.getSourceReference()
)
)
except AttributeError:
pass
# The module the "global" statement refers to.
module = provider.getParentModule()
# Can give multiple names.
for variable_name in node.names:
closure_variable = None
# Re-use already taken global variables, in order to avoid creating yet
# another instance, esp. as the indications could then potentially not
# be shared.
if provider.hasTakenVariable(variable_name):
closure_variable = provider.getTakenVariable(variable_name)
# Only global variables count. Could have a closure reference to
# a location of a parent function here.
if not closure_variable.isModuleVariable():
closure_variable = None
if closure_variable is None:
module_variable = module.getVariableForAssignment(
variable_name = variable_name
)
closure_variable = provider.addClosureVariable(
variable = module_variable
)
assert closure_variable.isModuleVariable()
if Utils.python_version < 340 and \
provider.isClassDictCreation() and \
closure_variable.getName() == "__class__":
SyntaxErrors.raiseSyntaxError(
reason = "cannot make __class__ global",
source_ref = source_ref
)
provider.registerProvidedVariable(
variable = closure_variable
)
return None
def buildTryExceptionNode(provider, node, source_ref):
# Try/except nodes. Re-formulated as described in the developer
# manual. Exception handlers made the assignment to variables explicit. Same
# for the "del" as done for Python3. Also catches always work a tuple of
# exception types and hides away that they may be built or not.
# Many variables and branches, due to the re-formulation that is going on
# here, which just has the complexity, pylint: disable=R0912,R0914
tried = buildStatementsNode(
provider = provider,
nodes = node.body,
source_ref = source_ref
)
handlers = []
for handler in node.handlers:
exception_expression, exception_assign, exception_block = (
handler.type,
handler.name,
handler.body
)
if exception_assign is None:
statements = [
buildStatementsNode(
provider = provider,
nodes = exception_block,
source_ref = source_ref
)
]
elif Utils.python_version < 300:
statements = [
buildAssignmentStatements(
provider = provider,
node = exception_assign,
source = ExpressionCaughtExceptionValueRef(
source_ref = source_ref.atInternal()
),
source_ref = source_ref.atInternal()
),
buildStatementsNode(
provider = provider,
nodes = exception_block,
source_ref = source_ref
)
]
else:
target_info = decodeAssignTarget(
provider = provider,
node = exception_assign,
source_ref = source_ref,
)
# We didn't allow None, therefore it cannot be None, and
# the unpack is safe: pylint: disable=W0633
kind, detail = target_info
assert kind == "Name", kind
kind = "Name_Exception"
statements = [
buildAssignmentStatements(
provider = provider,
node = exception_assign,
source = ExpressionCaughtExceptionValueRef(
source_ref = source_ref.atInternal()
),
source_ref = source_ref.atInternal()
),
makeTryFinallyStatement(
tried = buildStatementsNode(
provider = provider,
nodes = exception_block,
source_ref = source_ref
),
final = buildDeleteStatementFromDecoded(
kind = kind,
detail = detail,
source_ref = source_ref
),
source_ref = source_ref
)
]
handler_body = makeStatementsSequence(
statements = statements,
allow_none = True,
source_ref = source_ref
)
exception_types = buildNode(
provider = provider,
node = exception_expression,
source_ref = source_ref,
allow_none = True
)
# The exception types should be a tuple, so as to be most general.
if exception_types is None:
if handler is not node.handlers[-1]:
SyntaxErrors.raiseSyntaxError(
reason = "default 'except:' must be last",
source_ref = source_ref.atLineNumber(
handler.lineno-1
if Options.isFullCompat() else
handler.lineno
)
)
handlers.append(
(
exception_types,
handler_body,
)
)
# Reraise by default
exception_handling = makeReraiseExceptionStatement(
source_ref = source_ref
)
for exception_type, handler in reversed(handlers):
if exception_type is None:
# A default handler was given, so use that indead.
exception_handling = handler
else:
exception_handling = StatementsSequence(
statements = (
StatementConditional(
condition = ExpressionComparisonExceptionMatch(
left = ExpressionCaughtExceptionTypeRef(
source_ref = exception_type.source_ref
),
right = exception_type,
source_ref = exception_type.source_ref
),
yes_branch = handler,
no_branch = exception_handling,
source_ref = exception_type.source_ref
),
),
source_ref = exception_type.source_ref
)
prelude = (
StatementPreserveFrameException(
source_ref = source_ref.atInternal()
),
StatementPublishException(
source_ref = source_ref.atInternal()
)
)
if exception_handling is None:
# For Python3, we need not publish at all, if all we do is to revert
# that immediately. For Python2, the publish may release previously
# published exception, which has side effects potentially.
if Utils.python_version < 300:
exception_handling = StatementsSequence(
statements = prelude,
source_ref = source_ref.atInternal()
)
public_exc = True
else:
public_exc = False
else:
public_exc = True
if Utils.python_version < 300:
exception_handling.setStatements(
prelude + exception_handling.getStatements()
)
else:
exception_handling = StatementsSequence(
statements = prelude + (
makeTryFinallyStatement(
tried = exception_handling,
final = StatementRestoreFrameException(
source_ref = source_ref.atInternal()
),
source_ref = source_ref
),
),
source_ref = source_ref.atInternal()
)
no_raise = buildStatementsNode(
provider = provider,
nodes = node.orelse,
source_ref = source_ref
)
if no_raise is None:
return StatementTryExcept(
tried = tried,
handling = exception_handling,
public_exc = public_exc,
source_ref = source_ref
)
else:
return makeTryExceptNoRaise(
provider = provider,
temp_scope = provider.allocateTempScope("try_except"),
handling = exception_handling,
tried = tried,
public_exc = public_exc,
no_raise = no_raise,
source_ref = source_ref
)
def buildFunctionWithParsing(provider, function_kind, name, function_doc, flags,
node, source_ref):
# This contains a complex re-formulation for nested parameter functions.
# pylint: disable=R0914
kind = getKind(node)
assert kind in ("FunctionDef", "Lambda", "AsyncFunctionDef"), "unsupported for kind " + kind
def extractArg(arg):
if arg is None:
return None
elif type(arg) is str:
return mangleName(arg, provider)
elif getKind(arg) == "Name":
return mangleName(arg.id, provider)
elif getKind(arg) == "arg":
return mangleName(arg.arg, provider)
elif getKind(arg) == "Tuple":
# These are to be re-formulated on the outside.
assert False
else:
assert False, getKind(arg)
special_args = {}
def extractNormalArgs(args):
normal_args = []
for arg in args:
if type(arg) is not str and getKind(arg) == "Tuple":
special_arg_name = ".%d" % (len(special_args) + 1)
special_args[special_arg_name] = arg.elts
normal_args.append(special_arg_name)
else:
normal_args.append(extractArg(arg))
return normal_args
normal_args = extractNormalArgs(node.args.args)
parameters = ParameterSpec(
ps_name = name,
ps_normal_args = normal_args,
ps_kw_only_args = [
extractArg(arg)
for arg in
node.args.kwonlyargs
]
if python_version >= 300 else
[],
ps_list_star_arg = extractArg(node.args.vararg),
ps_dict_star_arg = extractArg(node.args.kwarg),
ps_default_count = len(node.args.defaults)
)
message = parameters.checkValid()
if message is not None:
SyntaxErrors.raiseSyntaxError(
message,
source_ref
)
code_object = CodeObjectSpec(
co_name = name,
co_kind = function_kind,
co_varnames = parameters.getParameterNames(),
co_argcount = parameters.getArgumentCount(),
co_kwonlyargcount = parameters.getKwOnlyParameterCount(),
co_has_starlist = parameters.getStarListArgumentName() is not None,
co_has_stardict = parameters.getStarDictArgumentName() is not None
)
outer_body = ExpressionFunctionBody(
provider = provider,
name = name,
flags = flags,
doc = function_doc,
parameters = parameters,
source_ref = source_ref
)
if special_args:
inner_name = name.strip("<>") + "$inner"
inner_arg_names = []
iter_vars = []
values = []
statements = []
def unpackFrom(source, arg_names):
accesses = []
sub_special_index = 0
iter_var = outer_body.allocateTempVariable(None, "arg_iter_%d" % len(iter_vars))
iter_vars.append(iter_var)
statements.append(
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = iter_var,
source_ref = source_ref
),
source = ExpressionBuiltinIter1(
value = source,
source_ref = source_ref
),
source_ref = source_ref
)
)
for element_index, arg_name in enumerate(arg_names):
if getKind(arg_name) == "Name":
inner_arg_names.append(arg_name.id)
arg_var = outer_body.allocateTempVariable(None, "tmp_" + arg_name.id)
statements.append(
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = arg_var,
source_ref = source_ref
),
source = ExpressionSpecialUnpack(
value = ExpressionTempVariableRef(
variable = iter_var,
source_ref = source_ref
),
count = element_index + 1,
expected = len(arg_names),
source_ref = source_ref
),
source_ref = source_ref
)
)
accesses.append(
ExpressionTempVariableRef(
variable = arg_var,
source_ref = source_ref
)
)
elif getKind(arg_name) == "Tuple":
accesses.extend(
unpackFrom(
source = ExpressionSpecialUnpack(
value = ExpressionTempVariableRef(
variable = iter_var,
source_ref = source_ref
),
count = element_index + 1,
expected = len(arg_names),
source_ref = source_ref
),
arg_names = arg_name.elts
)
)
sub_special_index += 1
else:
assert False, arg_name
statements.append(
StatementSpecialUnpackCheck(
iterator = ExpressionTempVariableRef(
variable = iter_var,
source_ref = source_ref
),
count = len(arg_names),
source_ref = source_ref
)
)
return accesses
for arg_name in parameters.getParameterNames():
if arg_name.startswith('.'):
source = ExpressionVariableRef(
variable_name = arg_name,
source_ref = source_ref
)
values.extend(
unpackFrom(source, special_args[arg_name])
)
else:
values.append(
ExpressionVariableRef(
variable_name = arg_name,
source_ref = source_ref
)
)
inner_arg_names.append(arg_name)
inner_parameters = ParameterSpec(
ps_name = inner_name,
ps_normal_args = inner_arg_names,
ps_kw_only_args = (),
ps_list_star_arg = None,
ps_dict_star_arg = None,
ps_default_count = None
)
function_body = ExpressionFunctionBody(
provider = outer_body,
name = inner_name,
flags = flags,
doc = function_doc,
parameters = inner_parameters,
source_ref = source_ref
)
statements.append(
StatementReturn(
ExpressionFunctionCall(
function = ExpressionFunctionCreation(
function_ref = ExpressionFunctionRef(
function_body = function_body,
source_ref = source_ref
),
code_object = code_object,
defaults = (),
kw_defaults = None,
annotations = None,
source_ref = source_ref
),
values = values,
source_ref = source_ref
),
source_ref = source_ref
)
)
outer_body.setBody(
makeStatementsSequenceFromStatement(
statement = makeTryFinallyStatement(
provider,
tried = statements,
final = [
StatementReleaseVariable(
variable = variable,
source_ref = source_ref
)
for variable in
outer_body.getTempVariables()
] ,
source_ref = source_ref,
public_exc = False
)
)
)
else:
function_body = outer_body
return outer_body, function_body, code_object
def onEnterNode(self, node):
# Mighty complex code with lots of branches and statements, but it
# couldn't be less without making it more difficult.
# pylint: disable=R0912,R0915
if node.isExpressionTargetVariableRef():
provider = node.getParentVariableProvider()
variable = node.getVariable()
if variable is None:
variable_name = node.getVariableName()
variable = provider.getVariableForAssignment(
variable_name=variable_name)
node.setVariable(variable)
variable.addVariableUser(provider)
elif node.isExpressionVariableRef():
if node.getVariable() is None:
provider = node.getParentVariableProvider()
if provider.isEarlyClosure():
variable = provider.getVariableForReference(
variable_name=node.getVariableName())
# Python3.4 version respects closure variables taken can be
# overridden by writes to locals. It should be done for
# globals too, on all versions, but for Python2 the locals
# dictionary is avoided unless "exec" appears, so it's not
# done.
if variable.getOwner() is not provider:
if python_version >= 340 or \
(python_version >= 300 and \
variable.isModuleVariable()):
variable = Variables.MaybeLocalVariable(
owner=provider, maybe_variable=variable)
node.setVariable(variable)
elif node.isExpressionTempVariableRef():
if node.getVariable().getOwner() != node.getParentVariableProvider(
):
node.setVariable(
node.getParentVariableProvider().addClosureVariable(
node.getVariable()))
elif node.isExpressionGeneratorObjectBody():
self._handleNonLocal(node)
# Python3.4 allows for class declarations to be made global, even
# after they were declared, so we need to fix this up.
# TODO: Then this may not even have to be here at all.
if python_version >= 340:
self._handleQualnameSetup(node)
elif node.isExpressionCoroutineObjectBody():
self._handleNonLocal(node)
# TODO: Then this may not even have to be here at all.
self._handleQualnameSetup(node)
elif node.isExpressionClassBody():
self._handleNonLocal(node)
# Python3.4 allows for class declarations to be made global, even
# after they were declared, so we need to fix this up.
if python_version >= 340:
self._handleQualnameSetup(node)
elif node.isExpressionFunctionBody():
self._handleNonLocal(node)
# Python 3.4 allows for class declarations to be made global, even
# after they were declared, so we need to fix this up.
if python_version >= 340:
self._handleQualnameSetup(node)
# Attribute access of names of class functions should be mangled, if
# they start with "__", but do not end in "__" as well.
elif node.isExpressionAttributeLookup() or \
node.isStatementAssignmentAttribute() or \
node.isStatementDelAttribute():
attribute_name = node.getAttributeName()
if attribute_name.startswith("__") and \
not attribute_name.endswith("__"):
seen_function = False
current = node
while True:
current = current.getParentVariableProvider()
if current.isCompiledPythonModule():
break
if current.isExpressionClassBody():
if seen_function:
node.setAttributeName(
"_%s%s" % (current.getName().lstrip('_'),
attribute_name))
break
else:
seen_function = True
# Check if continue and break are properly in loops. If not, raise a
# syntax error.
elif node.isStatementLoopBreak() or node.isStatementLoopContinue():
current = node
while True:
if current.isParentVariableProvider():
if node.isStatementLoopContinue():
message = "'continue' not properly in loop"
col_offset = 16 if python_version >= 300 else None
display_line = True
source_line = None
else:
message = "'break' outside loop"
if isFullCompat():
col_offset = 2 if python_version >= 300 else None
display_line = True
source_line = "" if python_version >= 300 else None
else:
col_offset = 13
display_line = True
source_line = None
SyntaxErrors.raiseSyntaxError(
message,
source_ref=node.getSourceReference(),
col_offset=col_offset,
display_line=display_line,
source_line=source_line)
current = current.getParent()
if current.isStatementLoop():
break
def buildReturnNode(provider, node, source_ref):
if not provider.isExpressionFunctionBody() or \
provider.isClassDictCreation():
SyntaxErrors.raiseSyntaxError(
"'return' outside function",
source_ref,
None if Utils.python_version < 300 else (
node.col_offset
if provider.isPythonModule() else
node.col_offset+4
)
)
expression = buildNode(provider, node.value, source_ref, allow_none = True)
if expression is None:
expression = ExpressionConstantRef(
constant = None,
source_ref = source_ref,
user_provided = True
)
# Indicate exceptions to potentially try/finally structures.
indicator_statements = makeTryFinallyIndicatorStatements(
is_loop_exit = False,
source_ref = source_ref
)
if indicator_statements and expression.mayRaiseException(BaseException):
tmp_variable = provider.allocateTempVariable(
temp_scope = provider.allocateTempScope("return"),
name = "value"
)
statements = [
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = tmp_variable,
source_ref = expression.getSourceReference()
),
source = expression,
source_ref = source_ref
)
] + indicator_statements + [
StatementReturn(
expression = ExpressionTempVariableRef(
variable = tmp_variable,
source_ref = expression.getSourceReference()
),
source_ref = source_ref
)
]
return makeTryFinallyStatement(
tried = statements,
final = StatementReleaseVariable(
variable = tmp_variable,
tolerant = True,
source_ref = source_ref
),
source_ref = source_ref
)
else:
return makeStatementsSequenceOrStatement(
statements = indicator_statements + [
StatementReturn(
expression = expression,
source_ref = source_ref
)
],
source_ref = source_ref
)
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fulfill many options, leading to many branches and statements
# to deal with them. pylint: disable=too-many-branches
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir=os.path.dirname(os.path.abspath(filename)))
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = os.path.join(
os.path.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt")
if os.path.isfile(origin_prefix_filename):
data_files.append((filename, "orig-prefix.txt"))
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(filename=filename)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort. If
# we are not in full compat, and user has not specified the Python
# versions he wants, tell him about the potential version problem.
error_message = SyntaxErrors.formatOutput(e)
if not Options.isFullCompat() and \
Options.getIntendedPythonVersion() is None:
if python_version < 300:
suggested_python_version_str = getSupportedPythonVersions()[-1]
else:
suggested_python_version_str = "2.7"
error_message += """
Nuitka is very syntax compatible with standard Python. It is currently running
with Python version '%s', you might want to specify more clearly with the use
of e.g. '--python-version=%s' option, if that's not the one expected.
""" % (python_version_str, suggested_python_version_str)
sys.exit(error_message)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(main_module=main_module)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
removeDirectory(path=getSourceDirectoryPath(main_module),
ignore_errors=False)
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(0, (None, binary_filename, None))
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
for module in ModuleRegistry.getUncompiledModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module))
copyUsedDLLs(dist_dir=dist_dir,
standalone_entry_points=standalone_entry_points)
for module in ModuleRegistry.getDoneModules():
data_files.extend(Plugins.considerDataFiles(module))
for source_filename, target_filename in data_files:
target_filename = os.path.join(
getStandaloneDirectoryPath(main_module), target_filename)
makePath(os.path.dirname(target_filename))
shutil.copy2(source_filename, target_filename)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(("chmod", "-x", getResultFullpath(main_module)))
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree=main_module,
clean_path=Options.shallClearPythonPathEnvironment())
else:
executeMain(
binary_filename=getResultFullpath(main_module),
clean_path=Options.shallClearPythonPathEnvironment())
def main():
""" Main program flow of Nuitka
At this point, options will be parsed already, Nuitka will be executing
in the desired version of Python with desired flags, and we just get
to execute the task assigned.
We might be asked to only re-compile generated C++, dump only an XML
representation of the internal node tree after optimization, etc.
"""
# Main has to fullfil many options, leading to many branches and statements
# to deal with them. pylint: disable=R0912
positional_args = Options.getPositionalArgs()
assert len(positional_args) > 0
filename = Options.getPositionalArgs()[0]
# Inform the importing layer about the main script directory, so it can use
# it when attempting to follow imports.
Importing.setMainScriptDirectory(
main_dir = Utils.dirname(Utils.abspath(filename))
)
# Detect to be frozen modules if any, so we can consider to not recurse
# to them.
if Options.isStandaloneMode():
for module in detectEarlyImports():
ModuleRegistry.addUncompiledModule(module)
if module.getName() == "site":
origin_prefix_filename = Utils.joinpath(
Utils.dirname(module.getCompileTimeFilename()),
"orig-prefix.txt"
)
if Utils.isFile(origin_prefix_filename):
data_files.append(
(filename, "orig-prefix.txt")
)
# Turn that source code into a node tree structure.
try:
main_module = createNodeTree(
filename = filename
)
except (SyntaxError, IndentationError) as e:
# Syntax or indentation errors, output them to the user and abort.
sys.exit(
SyntaxErrors.formatOutput(e)
)
if Options.shallDumpBuiltTreeXML():
for module in ModuleRegistry.getDoneModules():
dumpTreeXML(module)
elif Options.shallDisplayBuiltTree():
displayTree(main_module)
else:
result, options = compileTree(
main_module = main_module
)
# Exit if compilation failed.
if not result:
sys.exit(1)
if Options.shallNotDoExecCppCall():
sys.exit(0)
# Remove the source directory (now build directory too) if asked to.
if Options.isRemoveBuildDir():
shutil.rmtree(
getSourceDirectoryPath(main_module)
)
if Options.isStandaloneMode():
binary_filename = options["result_name"] + ".exe"
standalone_entry_points.insert(
0,
(binary_filename, None)
)
dist_dir = getStandaloneDirectoryPath(main_module)
for module in ModuleRegistry.getDoneUserModules():
standalone_entry_points.extend(
Plugins.considerExtraDlls(dist_dir, module)
)
if Utils.getOS() == "NetBSD":
warning("Standalone mode on NetBSD is not functional, due to $ORIGIN linkage not being supported.")
copyUsedDLLs(
dist_dir = dist_dir,
standalone_entry_points = standalone_entry_points
)
for source_filename, target_filename in data_files:
shutil.copy2(
source_filename,
Utils.joinpath(
getStandaloneDirectoryPath(main_module),
target_filename
)
)
# Modules should not be executable, but Scons creates them like it, fix
# it up here.
if Utils.getOS() != "Windows" and Options.shallMakeModule():
subprocess.call(
(
"chmod",
"-x",
getResultFullpath(main_module)
)
)
# Execute the module immediately if option was given.
if Options.shallExecuteImmediately():
if Options.shallMakeModule():
executeModule(
tree = main_module,
clean_path = Options.shallClearPythonPathEnvironment()
)
else:
executeMain(
binary_filename = getResultFullpath(main_module),
clean_path = Options.shallClearPythonPathEnvironment()
)
def buildImportFromNode(provider, node, source_ref):
# "from .. import .." statements. This may trigger a star import, or
# multiple names being looked up from the given module variable name.
# This is pretty complex, pylint: disable=R0912,R0914
module_name = node.module if node.module is not None else ""
level = node.level
# Importing from "__future__" module may enable flags to the parser,
# that we need to know about, handle that.
if module_name == "__future__":
_handleFutureImport(provider, node, source_ref)
target_names = []
import_names = []
# Mapping imported "fromlist" to assigned "fromlist" if any, handling the
# star case as well.
for import_desc in node.names:
object_name, local_name = import_desc.name, import_desc.asname
if object_name == '*':
target_names.append(None)
assert local_name is None
else:
target_names.append(
local_name
if local_name is not None else
object_name
)
import_names.append(object_name)
# Star imports get special treatment.
if None in target_names:
# More than "*" is a syntax error in Python, need not care about this at
# all, it's only allowed value for import list in this case.
assert target_names == [None]
# Python3 made it so that these can only occur on the module level,
# so this a syntax error if not there. For Python2 it is OK to
# occur everywhere though.
if not provider.isCompiledPythonModule() and python_version >= 300:
SyntaxErrors.raiseSyntaxError(
"import * only allowed at module level",
provider.getSourceReference()
)
# Functions with star imports get a marker.
if provider.isExpressionFunctionBody():
provider.markAsStarImportContaining()
return StatementImportStar(
module_import = ExpressionImportModule(
module_name = module_name,
import_list = ('*',),
level = level,
source_ref = source_ref
),
source_ref = source_ref
)
else:
def makeImportName(import_name):
if module_name == "__future__":
# Make "__future__" imports tie hard immediately, they cannot be
# any other way.
return ExpressionImportModuleHard(
module_name = "__future__",
import_name = import_name,
source_ref = source_ref
)
else:
# Refer to be module, or a clone of the reference if need be.
return ExpressionImportName(
module = imported_from_module,
import_name = import_name,
source_ref = source_ref
)
imported_from_module = ExpressionImportModule(
module_name = module_name,
import_list = tuple(import_names),
level = level,
source_ref = source_ref
)
# If we have multiple names to import, consider each.
multi_names = len(target_names) > 1
statements = []
if multi_names:
tmp_import_from = provider.allocateTempVariable(
temp_scope = provider.allocateTempScope("import_from"),
name = "module"
)
statements.append(
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = tmp_import_from,
source_ref = source_ref
),
source = imported_from_module,
source_ref = source_ref
)
)
imported_from_module = ExpressionTempVariableRef(
variable = tmp_import_from,
source_ref = source_ref
)
import_statements = []
first = True
for target_name, import_name in zip(target_names, import_names):
# Make a clone of the variable reference, if we are going to use
# another one.
if not first:
imported_from_module = imported_from_module.makeClone()
first = False
import_statements.append(
StatementAssignmentVariable(
variable_ref = ExpressionTargetVariableRef(
variable_name = mangleName(target_name, provider),
source_ref = source_ref
),
source = makeImportName(
import_name = import_name,
),
source_ref = source_ref
)
)
# Release the temporary module value as well.
if multi_names:
statements.append(
makeTryFinallyStatement(
provider = provider,
tried = import_statements,
final = (
StatementReleaseVariable(
variable = tmp_import_from,
source_ref = source_ref
),
),
source_ref = source_ref
)
)
else:
statements.extend(import_statements)
# Note: Each import is sequential. It can succeed, and the failure of a
# later one is not undoing previous ones. We can therefore have a
# sequence of imports that each only import one thing therefore.
return StatementsSequence(
statements = mergeStatements(statements),
source_ref = source_ref
)
def buildFunctionWithParsing(provider, function_kind, name, function_doc,
flags, node, source_ref):
# This contains a complex re-formulation for nested parameter functions.
# pylint: disable=R0914
kind = getKind(node)
assert kind in ("FunctionDef", "Lambda",
"AsyncFunctionDef"), "unsupported for kind " + kind
def extractArg(arg):
if arg is None:
return None
elif type(arg) is str:
return mangleName(arg, provider)
elif getKind(arg) == "Name":
return mangleName(arg.id, provider)
elif getKind(arg) == "arg":
return mangleName(arg.arg, provider)
elif getKind(arg) == "Tuple":
# These are to be re-formulated on the outside.
assert False
else:
assert False, getKind(arg)
special_args = {}
def extractNormalArgs(args):
normal_args = []
for arg in args:
if type(arg) is not str and getKind(arg) == "Tuple":
special_arg_name = ".%d" % (len(special_args) + 1)
special_args[special_arg_name] = arg.elts
normal_args.append(special_arg_name)
else:
normal_args.append(extractArg(arg))
return normal_args
normal_args = extractNormalArgs(node.args.args)
parameters = ParameterSpec(
ps_name=name,
ps_normal_args=normal_args,
ps_kw_only_args=[extractArg(arg) for arg in node.args.kwonlyargs]
if python_version >= 300 else [],
ps_list_star_arg=extractArg(node.args.vararg),
ps_dict_star_arg=extractArg(node.args.kwarg),
ps_default_count=len(node.args.defaults))
message = parameters.checkValid()
if message is not None:
SyntaxErrors.raiseSyntaxError(message, source_ref)
code_object = CodeObjectSpec(
co_name=name,
co_kind=function_kind,
co_varnames=parameters.getParameterNames(),
co_argcount=parameters.getArgumentCount(),
co_kwonlyargcount=parameters.getKwOnlyParameterCount(),
co_has_starlist=parameters.getStarListArgumentName() is not None,
co_has_stardict=parameters.getStarDictArgumentName() is not None)
outer_body = ExpressionFunctionBody(provider=provider,
name=name,
flags=flags,
doc=function_doc,
parameters=parameters,
source_ref=source_ref)
if special_args:
inner_name = name.strip("<>") + "$inner"
inner_arg_names = []
iter_vars = []
values = []
statements = []
def unpackFrom(source, arg_names):
accesses = []
sub_special_index = 0
iter_var = outer_body.allocateTempVariable(
None, "arg_iter_%d" % len(iter_vars))
iter_vars.append(iter_var)
statements.append(
StatementAssignmentVariable(
variable_ref=ExpressionTargetTempVariableRef(
variable=iter_var, source_ref=source_ref),
source=ExpressionBuiltinIter1(value=source,
source_ref=source_ref),
source_ref=source_ref))
for element_index, arg_name in enumerate(arg_names):
if getKind(arg_name) == "Name":
inner_arg_names.append(arg_name.id)
arg_var = outer_body.allocateTempVariable(
None, "tmp_" + arg_name.id)
statements.append(
StatementAssignmentVariable(
variable_ref=ExpressionTargetTempVariableRef(
variable=arg_var, source_ref=source_ref),
source=ExpressionSpecialUnpack(
value=ExpressionTempVariableRef(
variable=iter_var, source_ref=source_ref),
count=element_index + 1,
expected=len(arg_names),
source_ref=source_ref),
source_ref=source_ref))
accesses.append(
ExpressionTempVariableRef(variable=arg_var,
source_ref=source_ref))
elif getKind(arg_name) == "Tuple":
accesses.extend(
unpackFrom(source=ExpressionSpecialUnpack(
value=ExpressionTempVariableRef(
variable=iter_var, source_ref=source_ref),
count=element_index + 1,
expected=len(arg_names),
source_ref=source_ref),
arg_names=arg_name.elts))
sub_special_index += 1
else:
assert False, arg_name
statements.append(
StatementSpecialUnpackCheck(iterator=ExpressionTempVariableRef(
variable=iter_var, source_ref=source_ref),
count=len(arg_names),
source_ref=source_ref))
return accesses
for arg_name in parameters.getParameterNames():
if arg_name.startswith('.'):
source = ExpressionVariableRef(variable_name=arg_name,
source_ref=source_ref)
values.extend(unpackFrom(source, special_args[arg_name]))
else:
values.append(
ExpressionVariableRef(variable_name=arg_name,
source_ref=source_ref))
inner_arg_names.append(arg_name)
inner_parameters = ParameterSpec(ps_name=inner_name,
ps_normal_args=inner_arg_names,
ps_kw_only_args=(),
ps_list_star_arg=None,
ps_dict_star_arg=None,
ps_default_count=None)
function_body = ExpressionFunctionBody(provider=outer_body,
name=inner_name,
flags=flags,
doc=function_doc,
parameters=inner_parameters,
source_ref=source_ref)
statements.append(
StatementReturn(ExpressionFunctionCall(
function=ExpressionFunctionCreation(
function_ref=ExpressionFunctionRef(
function_body=function_body, source_ref=source_ref),
code_object=code_object,
defaults=(),
kw_defaults=None,
annotations=None,
source_ref=source_ref),
values=values,
source_ref=source_ref),
source_ref=source_ref))
outer_body.setBody(
makeStatementsSequenceFromStatement(
statement=makeTryFinallyStatement(
provider,
tried=statements,
final=[
StatementReleaseVariable(variable=variable,
source_ref=source_ref)
for variable in outer_body.getTempVariables()
],
source_ref=source_ref,
public_exc=False)))
else:
function_body = outer_body
return outer_body, function_body, code_object
def onEnterNode(self, node):
# Mighty complex code with lots of branches and statements, but it
# couldn't be less without making it more difficult.
# pylint: disable=R0912,R0915
if node.isExpressionTargetVariableRef():
provider = node.getParentVariableProvider()
if node.getVariable() is None:
variable_name = node.getVariableName()
variable = provider.getVariableForAssignment(
variable_name = variable_name
)
node.setVariable(variable)
addVariableUsage(node.getVariable(), provider)
elif node.isExpressionTargetTempVariableRef():
provider = node.getParentVariableProvider()
addVariableUsage(node.getVariable(), provider)
elif node.isExpressionVariableRef():
if node.getVariable() is None:
provider = node.getParentVariableProvider()
if provider.isEarlyClosure():
variable = provider.getVariableForReference(
variable_name = node.getVariableName()
)
# Python3.4 version respects closure variables taken can be
# overridden by writes to locals. It should be done for
# globals too, on all versions, but for Python2 the locals
# dictionary is avoided unless "exec" appears, so it's not
# done.
if variable.getOwner() is not provider:
if python_version >= 340 or \
(python_version >= 300 and \
variable.isModuleVariable()):
variable = Variables.MaybeLocalVariable(
owner = provider,
maybe_variable = variable
)
node.setVariable(variable)
elif node.isExpressionTempVariableRef():
if node.getVariable().getOwner() != node.getParentVariableProvider():
node.setVariable(
node.getParentVariableProvider().addClosureVariable(
node.getVariable()
)
)
elif node.isExpressionGeneratorObjectBody():
self._handleNonLocal(node)
# Python3.4 allows for class declarations to be made global, even
# after they were declared, so we need to fix this up.
# TODO: Then this may not even have to be here at all.
if python_version >= 340:
self._handleQualnameSetup(node)
elif node.isExpressionCoroutineObjectBody():
self._handleNonLocal(node)
# TODO: Then this may not even have to be here at all.
self._handleQualnameSetup(node)
elif node.isExpressionClassBody():
self._handleNonLocal(node)
# Python3.4 allows for class declarations to be made global, even
# after they were declared, so we need to fix this up.
if python_version >= 340:
self._handleQualnameSetup(node)
elif node.isExpressionFunctionBody():
self._handleNonLocal(node)
for variable in node.getParameters().getAllVariables():
addVariableUsage(variable, node)
# Python3.4 allows for class declarations to be made global, even
# after they were declared, so we need to fix this up.
if python_version >= 340:
self._handleQualnameSetup(node)
# Attribute access of names of class functions should be mangled, if
# they start with "__", but do not end in "__" as well.
elif node.isExpressionAttributeLookup() or \
node.isStatementAssignmentAttribute() or \
node.isStatementDelAttribute():
attribute_name = node.getAttributeName()
if attribute_name.startswith("__") and \
not attribute_name.endswith("__"):
seen_function = False
current = node
while True:
current = current.getParentVariableProvider()
if current.isCompiledPythonModule():
break
if current.isExpressionClassBody():
if seen_function:
node.setAttributeName(
"_%s%s" % (
current.getName().lstrip('_'),
attribute_name
)
)
break
else:
seen_function = True
# Check if continue and break are properly in loops. If not, raise a
# syntax error.
elif node.isStatementLoopBreak() or node.isStatementLoopContinue():
current = node
while True:
if current.isParentVariableProvider():
if node.isStatementLoopContinue():
message = "'continue' not properly in loop"
col_offset = 16 if python_version >= 300 else None
display_line = True
source_line = None
else:
message = "'break' outside loop"
if isFullCompat():
col_offset = 2 if python_version >= 300 else None
display_line = True
source_line = "" if python_version >= 300 else None
else:
col_offset = 13
display_line = True
source_line = None
SyntaxErrors.raiseSyntaxError(
message,
source_ref = node.getSourceReference(),
col_offset = col_offset,
display_line = display_line,
source_line = source_line
)
current = current.getParent()
if current.isStatementLoop():
break
def buildImportFromNode(provider, node, source_ref):
# "from .. import .." statements. This may trigger a star import, or
# multiple names being looked up from the given module variable name.
module_name = node.module if node.module is not None else ""
level = node.level
# Importing from "__future__" module may enable flags to the parser,
# that we need to know.
if module_name == "__future__":
# Future imports we see are all legal, and known to work.
if not provider.isCompiledPythonModule():
SyntaxErrors.raiseSyntaxError(
reason = """\
from __future__ imports must occur at the beginning of the file""",
col_offset = 8
if Utils.python_version >= 300 or \
not Options.isFullCompat()
else None,
source_ref = source_ref
)
for import_desc in node.names:
object_name, _local_name = import_desc.name, import_desc.asname
enableFutureFeature(object_name=object_name,
future_spec=source_ref.getFutureSpec(),
source_ref=source_ref)
# Remember it for checks to be applied once module is complete.
node.source_ref = source_ref
_future_import_nodes.append(node)
target_names = []
import_names = []
for import_desc in node.names:
object_name, local_name = import_desc.name, import_desc.asname
if object_name == '*':
target_names.append(None)
else:
target_names.append(
local_name if local_name is not None else object_name)
import_names.append(object_name)
if None in target_names:
# More than "*" is a syntax error in Python, need not care about this at
# all, it's only allowed value for import list in this case.
assert target_names == [None]
# Python3 made this a syntax error unfortunately.
if not provider.isCompiledPythonModule(
) and Utils.python_version >= 300:
SyntaxErrors.raiseSyntaxError(
"import * only allowed at module level",
provider.getSourceReference())
if provider.isExpressionFunctionBody():
provider.markAsStarImportContaining()
return StatementImportStar(module_import=ExpressionImportModule(
module_name=module_name,
import_list=('*', ),
level=level,
source_ref=source_ref),
source_ref=source_ref)
else:
# Make __future__ imports "hard" immediately, they cannot be any other
# way.
def makeImportName(import_name):
if module_name == "__future__":
return ExpressionImportModuleHard(module_name="__future__",
import_name=import_name,
source_ref=source_ref)
else:
# TODO: This ought to use a temporary variable for multiple
# names, instead of importing multiple times.
return ExpressionImportName(module=ExpressionImportModule(
module_name=module_name,
import_list=tuple(import_names),
level=level,
source_ref=source_ref),
import_name=import_name,
source_ref=source_ref)
import_nodes = []
for target_name, import_name in zip(target_names, import_names):
import_nodes.append(
StatementAssignmentVariable(
variable_ref=ExpressionTargetVariableRef(
variable_name=mangleName(target_name, provider),
source_ref=source_ref),
source=makeImportName(import_name=import_name, ),
source_ref=source_ref))
# Note: Each import is sequential. It can succeed, and the failure of a
# later one is not changing one. We can therefore have a sequence of
# imports that only import one thing therefore.
return StatementsSequence(statements=import_nodes,
source_ref=source_ref)
def buildImportFromNode(provider, node, source_ref):
# "from .. import .." statements. This may trigger a star import, or
# multiple names being looked up from the given module variable name.
module_name = node.module if node.module is not None else ""
level = node.level
# Importing from "__future__" module may enable flags to the parser,
# that we need to know.
if module_name == "__future__":
# Future imports we see are all legal, and known to work.
if not provider.isPythonModule():
SyntaxErrors.raiseSyntaxError(
reason = """\
from __future__ imports must occur at the beginning of the file""",
col_offset = 8
if Utils.python_version >= 300 or \
not Options.isFullCompat()
else None,
source_ref = source_ref
)
for import_desc in node.names:
object_name, _local_name = import_desc.name, import_desc.asname
enableFutureFeature(
object_name = object_name,
future_spec = source_ref.getFutureSpec(),
source_ref = source_ref
)
# Remember it for checks to be applied once module is complete.
node.source_ref = source_ref
_future_import_nodes.append(node)
target_names = []
import_names = []
for import_desc in node.names:
object_name, local_name = import_desc.name, import_desc.asname
if object_name == '*':
target_names.append(None)
else:
target_names.append(
local_name
if local_name is not None else
object_name
)
import_names.append(object_name)
if None in target_names:
# More than "*" is a syntax error in Python, need not care about this at
# all, it's only allowed value for import list in this case.
assert target_names == [None]
# Python3 made this a syntax error unfortunately.
if not provider.isPythonModule() and Utils.python_version >= 300:
SyntaxErrors.raiseSyntaxError(
"import * only allowed at module level",
provider.getSourceReference()
)
if provider.isExpressionFunctionBody():
provider.markAsStarImportContaining()
return StatementImportStar(
module_import = ExpressionImportModule(
module_name = module_name,
import_list = ('*',),
level = level,
source_ref = source_ref
),
source_ref = source_ref
)
else:
# Make __future__ imports "hard" immediately, they cannot be any other
# way.
def makeImportName(import_name):
if module_name == "__future__":
return ExpressionImportModuleHard(
module_name = "__future__",
import_name = import_name,
source_ref = source_ref
)
else:
# TODO: This ought to use a temporary variable for multiple
# names, instead of importing multiple times.
return ExpressionImportName(
module = ExpressionImportModule(
module_name = module_name,
import_list = tuple(import_names),
level = level,
source_ref = source_ref
),
import_name = import_name,
source_ref = source_ref
)
import_nodes = []
for target_name, import_name in zip(target_names, import_names):
import_nodes.append(
StatementAssignmentVariable(
variable_ref = ExpressionTargetVariableRef(
variable_name = mangleName(target_name, provider),
source_ref = source_ref
),
source = makeImportName(
import_name = import_name,
),
source_ref = source_ref
)
)
# Note: Each import is sequential. It can succeed, and the failure of a
# later one is not changing one. We can therefore have a sequence of
# imports that only import one thing therefore.
return StatementsSequence(
statements = import_nodes,
source_ref = source_ref
)
