def _buildInplaceAssignAttributeNode(
provider,
lookup_source,
attribute_name,
tmp_variable1,
tmp_variable2,
operator,
expression,
source_ref,
):
# First assign the target value to a temporary variable.
preserve_to_tmp = StatementAssignmentVariable(
variable=tmp_variable1,
source=ExpressionAttributeLookup(
source=lookup_source.makeClone(),
attribute_name=attribute_name,
source_ref=source_ref,
),
source_ref=source_ref,
)
# Second assign the in-place result to a temporary variable
inplace_to_tmp = StatementAssignmentVariable(
variable=tmp_variable2,
source=makeExpressionOperationBinaryInplace(
operator=operator,
left=ExpressionTempVariableRef(variable=tmp_variable1,
source_ref=source_ref),
right=expression,
source_ref=source_ref,
),
source_ref=source_ref,
)
# Third, copy it back.
copy_back_from_tmp = makeTryFinallyStatement(
provider=provider,
tried=StatementAssignmentAttribute(
expression=lookup_source.makeClone(),
attribute_name=attribute_name,
source=ExpressionTempVariableRef(variable=tmp_variable2,
source_ref=source_ref),
source_ref=source_ref,
),
final=StatementReleaseVariable(variable=tmp_variable2,
source_ref=source_ref),
source_ref=source_ref,
)
return (
preserve_to_tmp,
# making sure the above temporary variable is deleted in any case.
makeTryFinallyStatement(
provider=provider,
tried=(inplace_to_tmp, copy_back_from_tmp),
final=StatementReleaseVariable(variable=tmp_variable1,
source_ref=source_ref),
source_ref=source_ref,
),
)
def _buildInplaceAssignAttributeNode(provider, lookup_source, attribute_name,
tmp_variable1, tmp_variable2, operator,
expression, source_ref):
# First assign the target value to a temporary variable.
preserve_to_tmp = StatementAssignmentVariable(
variable=tmp_variable1,
source=ExpressionAttributeLookup(source=lookup_source.makeClone(),
attribute_name=attribute_name,
source_ref=source_ref),
source_ref=source_ref)
# Second assign the in-place result to a temporary variable
inplace_to_tmp = StatementAssignmentVariable(
variable=tmp_variable2,
source=makeExpressionOperationBinaryInplace(
operator=operator,
left=ExpressionTempVariableRef(variable=tmp_variable1,
source_ref=source_ref),
right=expression,
source_ref=source_ref),
source_ref=source_ref)
# Third, copy it over, if the reference values change, i.e. IsNot is true.
copy_back_from_tmp = StatementConditional(
condition=ExpressionComparisonIsNOT(
left=ExpressionTempVariableRef(variable=tmp_variable1,
source_ref=source_ref),
right=ExpressionTempVariableRef(variable=tmp_variable2,
source_ref=source_ref),
source_ref=source_ref),
yes_branch=makeStatementsSequenceFromStatement(
statement=StatementAssignmentAttribute(
expression=lookup_source.makeClone(),
attribute_name=attribute_name,
source=ExpressionTempVariableRef(variable=tmp_variable2,
source_ref=source_ref),
source_ref=source_ref)),
no_branch=None,
source_ref=source_ref)
copy_back_from_tmp = makeTryFinallyStatement(
provider=provider,
tried=copy_back_from_tmp,
final=StatementReleaseVariable(variable=tmp_variable2,
source_ref=source_ref),
source_ref=source_ref)
return (
preserve_to_tmp,
# making sure the above temporary variable is deleted in any case.
makeTryFinallyStatement(provider=provider,
tried=(
inplace_to_tmp,
copy_back_from_tmp,
),
final=StatementReleaseVariable(
variable=tmp_variable1,
source_ref=source_ref),
source_ref=source_ref))
def buildParseTree(provider, source_code, source_ref, is_module, is_main):
# There are a bunch of branches here, mostly to deal with version
# differences for module default variables. pylint: disable=too-many-branches
pushFutureSpec()
if is_module:
provider.future_spec = getFutureSpec()
body = parseSourceCodeToAst(
source_code=source_code,
filename=source_ref.getFilename(),
line_offset=source_ref.getLineNumber() - 1,
)
body, doc = extractDocFromBody(body)
if is_module and is_main and python_version >= 360:
provider.markAsNeedsAnnotationsDictionary()
result = buildStatementsNode(provider=provider, nodes=body, source_ref=source_ref)
checkFutureImportsOnlyAtStart(body)
internal_source_ref = source_ref.atInternal()
statements = []
if is_module:
# Add import of "site" module of main programs visibly in the node tree,
# so recursion and optimization can pick it up, checking its effects.
if is_main and "no_site" not in Options.getPythonFlags():
for path_imported_name in getPthImportedPackages():
statements.append(
StatementExpressionOnly(
expression=makeAbsoluteImportNode(
module_name=path_imported_name, source_ref=source_ref
),
source_ref=source_ref,
)
)
statements.append(
StatementExpressionOnly(
expression=makeAbsoluteImportNode(
module_name="site", source_ref=source_ref
),
source_ref=source_ref,
)
)
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__doc__",
source=makeConstantRefNode(
constant=doc, source_ref=internal_source_ref, user_provided=True
),
source_ref=internal_source_ref,
)
)
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__file__",
source=ExpressionModuleAttributeFileRef(
variable=provider.getVariableForReference("__file__"),
source_ref=internal_source_ref,
),
source_ref=internal_source_ref,
)
)
if provider.isCompiledPythonPackage():
# This assigns "__path__" value.
statements.append(createPathAssignment(provider, internal_source_ref))
statements.append(
createImporterCacheAssignment(provider, internal_source_ref)
)
if python_version >= 340 and not is_main:
statements += (
StatementAssignmentAttribute(
expression=ExpressionModuleAttributeSpecRef(
variable=provider.getVariableForReference("__spec__"),
source_ref=internal_source_ref,
),
attribute_name="origin",
source=ExpressionModuleAttributeFileRef(
variable=provider.getVariableForReference("__file__"),
source_ref=internal_source_ref,
),
source_ref=internal_source_ref,
),
StatementAssignmentAttribute(
expression=ExpressionModuleAttributeSpecRef(
variable=provider.getVariableForReference("__spec__"),
source_ref=internal_source_ref,
),
attribute_name="has_location",
source=makeConstantRefNode(True, internal_source_ref),
source_ref=internal_source_ref,
),
)
if provider.isCompiledPythonPackage():
statements.append(
StatementAssignmentAttribute(
expression=ExpressionModuleAttributeSpecRef(
variable=provider.getVariableForReference("__spec__"),
source_ref=internal_source_ref,
),
attribute_name="submodule_search_locations",
source=ExpressionVariableNameRef(
provider=provider,
variable_name="__path__",
source_ref=internal_source_ref,
),
source_ref=internal_source_ref,
)
)
if python_version >= 300:
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__cached__",
source=ExpressionConstantNoneRef(source_ref=internal_source_ref),
source_ref=internal_source_ref,
)
)
needs__initializing__ = not provider.isMainModule() and 300 <= python_version < 340
if needs__initializing__:
# Set "__initializing__" at the beginning to True
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__initializing__",
source=makeConstantRefNode(
constant=True, source_ref=internal_source_ref, user_provided=True
),
source_ref=internal_source_ref,
)
)
if provider.needsAnnotationsDictionary():
# Set "__annotations__" on module level to {}
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__annotations__",
source=makeConstantRefNode(
constant={}, source_ref=internal_source_ref, user_provided=True
),
source_ref=internal_source_ref,
)
)
# Now the module body if there is any at all.
if result is not None:
statements.extend(result.getStatements())
if needs__initializing__:
# Set "__initializing__" at the end to False
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__initializing__",
source=makeConstantRefNode(
constant=False, source_ref=internal_source_ref, user_provided=True
),
source_ref=internal_source_ref,
)
)
if is_module:
result = makeModuleFrame(
module=provider, statements=statements, source_ref=source_ref
)
popFutureSpec()
return result
else:
assert False
def buildAssignmentStatementsFromDecoded(provider, kind, detail, source,
source_ref):
# This is using many variable names on purpose, so as to give names to the
# unpacked detail values, and has many branches due to the many cases
# dealt with, pylint: disable=too-many-branches,too-many-locals,too-many-statements
if kind == "Name":
return StatementAssignmentVariableName(
provider=provider,
variable_name=detail,
source=source,
source_ref=source_ref,
)
elif kind == "Attribute":
lookup_source, attribute_name = detail
return StatementAssignmentAttribute(
expression=lookup_source,
attribute_name=mangleName(attribute_name, provider),
source=source,
source_ref=source_ref,
)
elif kind == "Subscript":
subscribed, subscript = detail
return StatementAssignmentSubscript(
expression=subscribed,
subscript=subscript,
source=source,
source_ref=source_ref,
)
elif kind == "Slice":
lookup_source, lower, upper = detail
# For Python3 there is no slicing operation, this is always done
# with subscript using a slice object. For Python2, it is only done
# if no "step" is provided.
use_sliceobj = python_version >= 300
if use_sliceobj:
return StatementAssignmentSubscript(
expression=lookup_source,
source=source,
subscript=makeExpressionBuiltinSlice(start=lower,
stop=upper,
step=None,
source_ref=source_ref),
source_ref=source_ref,
)
else:
return StatementAssignmentSlice(
expression=lookup_source,
lower=lower,
upper=upper,
source=source,
source_ref=source_ref,
)
elif kind == "Tuple":
temp_scope = provider.allocateTempScope("tuple_unpack")
source_iter_var = provider.allocateTempVariable(temp_scope=temp_scope,
name="source_iter")
element_vars = [
provider.allocateTempVariable(temp_scope=temp_scope,
name="element_%d" %
(element_index + 1))
for element_index in range(len(detail))
]
starred_list_var = None
starred_index = None
statements = []
for element_index, element in enumerate(detail):
if element[0] == "Starred":
if starred_index is not None:
raiseSyntaxError(
"two starred expressions in assignment"
if python_version < 390 else
"multiple starred expressions in assignment",
source_ref.atColumnNumber(0),
)
starred_index = element_index
for element_index, element in enumerate(detail):
element_var = element_vars[element_index]
if starred_list_var is not None:
statements.insert(
starred_index + 1,
StatementAssignmentVariable(
variable=element_var,
source=ExpressionListOperationPop(
list_arg=ExpressionTempVariableRef(
variable=starred_list_var,
source_ref=source_ref),
source_ref=source_ref,
),
source_ref=source_ref,
),
)
elif element[0] != "Starred":
statements.append(
StatementAssignmentVariable(
variable=element_var,
source=ExpressionSpecialUnpack(
value=ExpressionTempVariableRef(
variable=source_iter_var,
source_ref=source_ref),
count=element_index + 1,
expected=starred_index or len(detail),
starred=starred_index is not None,
source_ref=source_ref,
),
source_ref=source_ref,
))
else:
assert starred_index == element_index
starred_list_var = element_var
statements.append(
StatementAssignmentVariable(
variable=element_var,
source=ExpressionBuiltinList(
value=ExpressionTempVariableRef(
variable=source_iter_var,
source_ref=source_ref),
source_ref=source_ref,
),
source_ref=source_ref,
))
if starred_list_var is None:
statements.append(
StatementSpecialUnpackCheck(
iterator=ExpressionTempVariableRef(
variable=source_iter_var, source_ref=source_ref),
count=len(detail),
source_ref=source_ref,
))
else:
statements.insert(
starred_index + 1,
makeStatementConditional(
condition=makeComparisonExpression(
comparator="Lt",
left=ExpressionBuiltinLen(
value=ExpressionTempVariableRef(
variable=starred_list_var,
source_ref=source_ref),
source_ref=source_ref,
),
right=makeConstantRefNode(
constant=len(statements) - starred_index - 1,
source_ref=source_ref,
),
source_ref=source_ref,
),
yes_branch=makeRaiseExceptionExpressionFromTemplate(
exception_type="ValueError",
template="""\
not enough values to unpack (expected at least %d, got %%d)""" %
(len(statements) - 1),
template_args=makeBinaryOperationNode(
operator="Add",
left=ExpressionBuiltinLen(
value=ExpressionTempVariableRef(
variable=starred_list_var,
source_ref=source_ref),
source_ref=source_ref,
),
right=makeConstantRefNode(constant=starred_index,
source_ref=source_ref),
source_ref=source_ref,
),
source_ref=source_ref,
).asStatement(),
no_branch=None,
source_ref=source_ref,
),
)
if python_version >= 370:
iter_creation_class = ExpressionBuiltinIterForUnpack
else:
iter_creation_class = ExpressionBuiltinIter1
statements = [
StatementAssignmentVariable(
variable=source_iter_var,
source=iter_creation_class(value=source,
source_ref=source_ref),
source_ref=source_ref,
),
makeTryFinallyStatement(
provider=provider,
tried=statements,
final=(StatementReleaseVariable(variable=source_iter_var,
source_ref=source_ref), ),
source_ref=source_ref,
),
]
# When all is done, copy over to the actual assignment targets, starred
# or not makes no difference here anymore.
for element_index, element in enumerate(detail):
if element[0] == "Starred":
element = element[1]
element_var = element_vars[element_index]
statements.append(
buildAssignmentStatementsFromDecoded(
provider=provider,
kind=element[0],
detail=element[1],
source=ExpressionTempVariableRef(variable=element_var,
source_ref=source_ref),
source_ref=source_ref,
))
# Need to release temporary variables right after successful
# usage.
statements.append(
StatementDelVariable(variable=element_var,
tolerant=True,
source_ref=source_ref))
final_statements = []
for element_var in element_vars:
final_statements.append(
StatementReleaseVariable(variable=element_var,
source_ref=source_ref))
return makeTryFinallyStatement(
provider=provider,
tried=statements,
final=final_statements,
source_ref=source_ref,
)
elif kind == "Starred":
raiseSyntaxError(
"starred assignment target must be in a list or tuple",
source_ref.atColumnNumber(0),
)
else:
assert False, (kind, source_ref, detail)
def buildAssignmentStatementsFromDecoded(provider, kind, detail, source,
source_ref):
# This is using many variable names on purpose, so as to give names to the
# unpacked detail values, and has many branches due to the many cases
# dealt with, pylint: disable=too-many-branches,too-many-locals
if kind == "Name":
variable_ref = detail
return StatementAssignmentVariable(
variable_ref = variable_ref,
source = source,
source_ref = source_ref
)
elif kind == "Attribute":
lookup_source, attribute_name = detail
return StatementAssignmentAttribute(
expression = lookup_source,
attribute_name = attribute_name,
source = source,
source_ref = source_ref
)
elif kind == "Subscript":
subscribed, subscript = detail
return StatementAssignmentSubscript(
expression = subscribed,
subscript = subscript,
source = source,
source_ref = source_ref
)
elif kind == "Slice":
lookup_source, lower, upper = detail
# For Python3 there is no slicing operation, this is always done
# with subscript using a slice object. For Python2, it is only done
# if no "step" is provided.
use_sliceobj = python_version >= 300
if use_sliceobj:
return StatementAssignmentSubscript(
expression = lookup_source,
source = source,
subscript = ExpressionBuiltinSlice(
start = lower,
stop = upper,
step = None,
source_ref = source_ref
),
source_ref = source_ref
)
else:
return StatementAssignmentSlice(
expression = lookup_source,
lower = lower,
upper = upper,
source = source,
source_ref = source_ref
)
elif kind == "Tuple":
temp_scope = provider.allocateTempScope("tuple_unpack")
source_iter_var = provider.allocateTempVariable(
temp_scope = temp_scope,
name = "source_iter"
)
element_vars = [
provider.allocateTempVariable(
temp_scope = temp_scope,
name = "element_%d" % (
element_index + 1
)
)
for element_index in
range(len(detail))
]
starred_list_var = None
starred_index = None
statements = []
for element_index, element in enumerate(detail):
element_var = element_vars[element_index]
if starred_list_var is not None:
if element[0] == "Starred":
raiseSyntaxError(
"two starred expressions in assignment",
source_ref.atColumnNumber(0)
)
statements.insert(
starred_index+1,
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = element_var,
source_ref = source_ref
),
source = ExpressionListOperationPop(
list_arg = ExpressionTempVariableRef(
variable = starred_list_var,
source_ref = source_ref
),
source_ref = source_ref
),
source_ref = source_ref
)
)
elif element[0] != "Starred":
statements.append(
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = element_var,
source_ref = source_ref
),
source = ExpressionSpecialUnpack(
value = ExpressionTempVariableRef(
variable = source_iter_var,
source_ref = source_ref
),
count = element_index + 1,
expected = len(detail),
source_ref = source_ref
),
source_ref = source_ref
)
)
else:
starred_index = element_index
starred_list_var = element_var
statements.append(
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = element_var,
source_ref = source_ref
),
source = ExpressionBuiltinList(
value = ExpressionTempVariableRef(
variable = source_iter_var,
source_ref = source_ref
),
source_ref = source_ref
),
source_ref = source_ref
)
)
if starred_list_var is None:
statements.append(
StatementSpecialUnpackCheck(
iterator = ExpressionTempVariableRef(
variable = source_iter_var,
source_ref = source_ref
),
count = len(detail),
source_ref = source_ref
)
)
statements = [
StatementAssignmentVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = source_iter_var,
source_ref = source_ref
),
source = ExpressionBuiltinIter1(
value = source,
source_ref = source_ref
),
source_ref = source_ref
),
makeTryFinallyStatement(
provider = provider,
tried = statements,
final = (
StatementReleaseVariable(
variable = source_iter_var,
source_ref = source_ref
),
),
source_ref = source_ref
)
]
# When all is done, copy over to the actual assignment targets, starred
# or not makes no difference here anymore.
for element_index, element in enumerate(detail):
if element[0] == "Starred":
element = element[1]
element_var = element_vars[element_index]
statements.append(
buildAssignmentStatementsFromDecoded(
provider = provider,
kind = element[0],
detail = element[1],
source = ExpressionTempVariableRef(
variable = element_var,
source_ref = source_ref
),
source_ref = source_ref
)
)
# Need to release temporary variables right after successful
# usage.
statements.append(
StatementDelVariable(
variable_ref = ExpressionTargetTempVariableRef(
variable = element_var,
source_ref = source_ref
),
tolerant = True,
source_ref = source_ref,
)
)
final_statements = []
for element_var in element_vars:
final_statements.append(
StatementReleaseVariable(
variable = element_var,
source_ref = source_ref,
)
)
return makeTryFinallyStatement(
provider = provider,
tried = statements,
final = final_statements,
source_ref = source_ref
)
elif kind == "Starred":
raiseSyntaxError(
"starred assignment target must be in a list or tuple",
source_ref.atColumnNumber(0)
)
else:
assert False, (kind, source_ref, detail)
def buildParseTree(provider, ast_tree, source_ref, is_module, is_main):
# There are a bunch of branches here, mostly to deal with version
# differences for module default variables. pylint: disable=too-many-branches
# Maybe one day, we do exec inlining again, that is what this is for,
# then is_module won't be True, for now it always is.
pushFutureSpec()
if is_module:
provider.setFutureSpec(getFutureSpec())
body, doc = extractDocFromBody(ast_tree)
if is_module and is_main and python_version >= 0x360:
provider.markAsNeedsAnnotationsDictionary()
result = buildStatementsNode(provider=provider,
nodes=body,
source_ref=source_ref)
# After building, we can verify that all future statements were where they
# belong, namely at the start of the module.
checkFutureImportsOnlyAtStart(body)
internal_source_ref = source_ref.atInternal()
statements = []
if is_module:
# Add import of "site" module of main programs visibly in the node tree,
# so recursion and optimization can pick it up, checking its effects.
if is_main and not Options.hasPythonFlagNoSite():
statements.append(
StatementExpressionOnly(
expression=makeExpressionImportModuleFixed(
module_name="site", source_ref=source_ref),
source_ref=source_ref,
))
for path_imported_name in getPthImportedPackages():
if isHardModuleWithoutSideEffect(path_imported_name):
continue
statements.append(
StatementExpressionOnly(
expression=makeExpressionImportModuleFixed(
module_name=path_imported_name,
source_ref=source_ref),
source_ref=source_ref,
))
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__doc__",
source=makeConstantRefNode(constant=doc,
source_ref=internal_source_ref,
user_provided=True),
source_ref=internal_source_ref,
))
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__file__",
source=ExpressionModuleAttributeFileRef(
variable=provider.getVariableForReference("__file__"),
source_ref=internal_source_ref,
),
source_ref=internal_source_ref,
))
if provider.isCompiledPythonPackage():
# This assigns "__path__" value.
statements.append(
createPathAssignment(provider, internal_source_ref))
statements.append(
createImporterCacheAssignment(provider, internal_source_ref))
if python_version >= 0x340 and not is_main:
statements += (
StatementAssignmentAttribute(
expression=ExpressionModuleAttributeSpecRef(
variable=provider.getVariableForReference("__spec__"),
source_ref=internal_source_ref,
),
attribute_name="origin",
source=ExpressionModuleAttributeFileRef(
variable=provider.getVariableForReference("__file__"),
source_ref=internal_source_ref,
),
source_ref=internal_source_ref,
),
StatementAssignmentAttribute(
expression=ExpressionModuleAttributeSpecRef(
variable=provider.getVariableForReference("__spec__"),
source_ref=internal_source_ref,
),
attribute_name="has_location",
source=makeConstantRefNode(True, internal_source_ref),
source_ref=internal_source_ref,
),
)
if provider.isCompiledPythonPackage():
statements.append(
StatementAssignmentAttribute(
expression=ExpressionModuleAttributeSpecRef(
variable=provider.getVariableForReference(
"__spec__"),
source_ref=internal_source_ref,
),
attribute_name="submodule_search_locations",
source=ExpressionVariableNameRef(
provider=provider,
variable_name="__path__",
source_ref=internal_source_ref,
),
source_ref=internal_source_ref,
))
if python_version >= 0x300:
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__cached__",
source=ExpressionConstantNoneRef(
source_ref=internal_source_ref),
source_ref=internal_source_ref,
))
needs__initializing__ = (not provider.isMainModule()
and 0x300 <= python_version < 0x340)
if needs__initializing__:
# Set "__initializing__" at the beginning to True
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__initializing__",
source=makeConstantRefNode(constant=True,
source_ref=internal_source_ref,
user_provided=True),
source_ref=internal_source_ref,
))
if provider.needsAnnotationsDictionary():
# Set "__annotations__" on module level to {}
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__annotations__",
source=makeConstantRefNode(constant={},
source_ref=internal_source_ref,
user_provided=True),
source_ref=internal_source_ref,
))
# Now the module body if there is any at all.
if result is not None:
statements.extend(result.subnode_statements)
if needs__initializing__:
# Set "__initializing__" at the end to False
statements.append(
StatementAssignmentVariableName(
provider=provider,
variable_name="__initializing__",
source=makeConstantRefNode(constant=False,
source_ref=internal_source_ref,
user_provided=True),
source_ref=internal_source_ref,
))
if is_module:
result = makeModuleFrame(module=provider,
statements=statements,
source_ref=source_ref)
popFutureSpec()
return result
else:
assert False