def _buildInplaceAssignAttributeNode( provider, lookup_source, attribute_name, tmp_variable1, tmp_variable2, operator, expression, source_ref, ): # First assign the target value to a temporary variable. preserve_to_tmp = StatementAssignmentVariable( variable=tmp_variable1, source=ExpressionAttributeLookup( source=lookup_source.makeClone(), attribute_name=attribute_name, source_ref=source_ref, ), source_ref=source_ref, ) # Second assign the in-place result to a temporary variable inplace_to_tmp = StatementAssignmentVariable( variable=tmp_variable2, source=makeExpressionOperationBinaryInplace( operator=operator, left=ExpressionTempVariableRef(variable=tmp_variable1, source_ref=source_ref), right=expression, source_ref=source_ref, ), source_ref=source_ref, ) # Third, copy it back. copy_back_from_tmp = makeTryFinallyStatement( provider=provider, tried=StatementAssignmentAttribute( expression=lookup_source.makeClone(), attribute_name=attribute_name, source=ExpressionTempVariableRef(variable=tmp_variable2, source_ref=source_ref), source_ref=source_ref, ), final=StatementReleaseVariable(variable=tmp_variable2, source_ref=source_ref), source_ref=source_ref, ) return ( preserve_to_tmp, # making sure the above temporary variable is deleted in any case. makeTryFinallyStatement( provider=provider, tried=(inplace_to_tmp, copy_back_from_tmp), final=StatementReleaseVariable(variable=tmp_variable1, source_ref=source_ref), source_ref=source_ref, ), )
def _buildInplaceAssignAttributeNode(provider, lookup_source, attribute_name, tmp_variable1, tmp_variable2, operator, expression, source_ref): # First assign the target value to a temporary variable. preserve_to_tmp = StatementAssignmentVariable( variable=tmp_variable1, source=ExpressionAttributeLookup(source=lookup_source.makeClone(), attribute_name=attribute_name, source_ref=source_ref), source_ref=source_ref) # Second assign the in-place result to a temporary variable inplace_to_tmp = StatementAssignmentVariable( variable=tmp_variable2, source=makeExpressionOperationBinaryInplace( operator=operator, left=ExpressionTempVariableRef(variable=tmp_variable1, source_ref=source_ref), right=expression, source_ref=source_ref), source_ref=source_ref) # Third, copy it over, if the reference values change, i.e. IsNot is true. copy_back_from_tmp = StatementConditional( condition=ExpressionComparisonIsNOT( left=ExpressionTempVariableRef(variable=tmp_variable1, source_ref=source_ref), right=ExpressionTempVariableRef(variable=tmp_variable2, source_ref=source_ref), source_ref=source_ref), yes_branch=makeStatementsSequenceFromStatement( statement=StatementAssignmentAttribute( expression=lookup_source.makeClone(), attribute_name=attribute_name, source=ExpressionTempVariableRef(variable=tmp_variable2, source_ref=source_ref), source_ref=source_ref)), no_branch=None, source_ref=source_ref) copy_back_from_tmp = makeTryFinallyStatement( provider=provider, tried=copy_back_from_tmp, final=StatementReleaseVariable(variable=tmp_variable2, source_ref=source_ref), source_ref=source_ref) return ( preserve_to_tmp, # making sure the above temporary variable is deleted in any case. makeTryFinallyStatement(provider=provider, tried=( inplace_to_tmp, copy_back_from_tmp, ), final=StatementReleaseVariable( variable=tmp_variable1, source_ref=source_ref), source_ref=source_ref))
def buildParseTree(provider, source_code, source_ref, is_module, is_main): # There are a bunch of branches here, mostly to deal with version # differences for module default variables. pylint: disable=too-many-branches pushFutureSpec() if is_module: provider.future_spec = getFutureSpec() body = parseSourceCodeToAst( source_code=source_code, filename=source_ref.getFilename(), line_offset=source_ref.getLineNumber() - 1, ) body, doc = extractDocFromBody(body) if is_module and is_main and python_version >= 360: provider.markAsNeedsAnnotationsDictionary() result = buildStatementsNode(provider=provider, nodes=body, source_ref=source_ref) checkFutureImportsOnlyAtStart(body) internal_source_ref = source_ref.atInternal() statements = [] if is_module: # Add import of "site" module of main programs visibly in the node tree, # so recursion and optimization can pick it up, checking its effects. if is_main and "no_site" not in Options.getPythonFlags(): for path_imported_name in getPthImportedPackages(): statements.append( StatementExpressionOnly( expression=makeAbsoluteImportNode( module_name=path_imported_name, source_ref=source_ref ), source_ref=source_ref, ) ) statements.append( StatementExpressionOnly( expression=makeAbsoluteImportNode( module_name="site", source_ref=source_ref ), source_ref=source_ref, ) ) statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__doc__", source=makeConstantRefNode( constant=doc, source_ref=internal_source_ref, user_provided=True ), source_ref=internal_source_ref, ) ) statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__file__", source=ExpressionModuleAttributeFileRef( variable=provider.getVariableForReference("__file__"), source_ref=internal_source_ref, ), source_ref=internal_source_ref, ) ) if provider.isCompiledPythonPackage(): # This assigns "__path__" value. statements.append(createPathAssignment(provider, internal_source_ref)) statements.append( createImporterCacheAssignment(provider, internal_source_ref) ) if python_version >= 340 and not is_main: statements += ( StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference("__spec__"), source_ref=internal_source_ref, ), attribute_name="origin", source=ExpressionModuleAttributeFileRef( variable=provider.getVariableForReference("__file__"), source_ref=internal_source_ref, ), source_ref=internal_source_ref, ), StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference("__spec__"), source_ref=internal_source_ref, ), attribute_name="has_location", source=makeConstantRefNode(True, internal_source_ref), source_ref=internal_source_ref, ), ) if provider.isCompiledPythonPackage(): statements.append( StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference("__spec__"), source_ref=internal_source_ref, ), attribute_name="submodule_search_locations", source=ExpressionVariableNameRef( provider=provider, variable_name="__path__", source_ref=internal_source_ref, ), source_ref=internal_source_ref, ) ) if python_version >= 300: statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__cached__", source=ExpressionConstantNoneRef(source_ref=internal_source_ref), source_ref=internal_source_ref, ) ) needs__initializing__ = not provider.isMainModule() and 300 <= python_version < 340 if needs__initializing__: # Set "__initializing__" at the beginning to True statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__initializing__", source=makeConstantRefNode( constant=True, source_ref=internal_source_ref, user_provided=True ), source_ref=internal_source_ref, ) ) if provider.needsAnnotationsDictionary(): # Set "__annotations__" on module level to {} statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__annotations__", source=makeConstantRefNode( constant={}, source_ref=internal_source_ref, user_provided=True ), source_ref=internal_source_ref, ) ) # Now the module body if there is any at all. if result is not None: statements.extend(result.getStatements()) if needs__initializing__: # Set "__initializing__" at the end to False statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__initializing__", source=makeConstantRefNode( constant=False, source_ref=internal_source_ref, user_provided=True ), source_ref=internal_source_ref, ) ) if is_module: result = makeModuleFrame( module=provider, statements=statements, source_ref=source_ref ) popFutureSpec() return result else: assert False
def buildAssignmentStatementsFromDecoded(provider, kind, detail, source, source_ref): # This is using many variable names on purpose, so as to give names to the # unpacked detail values, and has many branches due to the many cases # dealt with, pylint: disable=too-many-branches,too-many-locals,too-many-statements if kind == "Name": return StatementAssignmentVariableName( provider=provider, variable_name=detail, source=source, source_ref=source_ref, ) elif kind == "Attribute": lookup_source, attribute_name = detail return StatementAssignmentAttribute( expression=lookup_source, attribute_name=mangleName(attribute_name, provider), source=source, source_ref=source_ref, ) elif kind == "Subscript": subscribed, subscript = detail return StatementAssignmentSubscript( expression=subscribed, subscript=subscript, source=source, source_ref=source_ref, ) elif kind == "Slice": lookup_source, lower, upper = detail # For Python3 there is no slicing operation, this is always done # with subscript using a slice object. For Python2, it is only done # if no "step" is provided. use_sliceobj = python_version >= 300 if use_sliceobj: return StatementAssignmentSubscript( expression=lookup_source, source=source, subscript=makeExpressionBuiltinSlice(start=lower, stop=upper, step=None, source_ref=source_ref), source_ref=source_ref, ) else: return StatementAssignmentSlice( expression=lookup_source, lower=lower, upper=upper, source=source, source_ref=source_ref, ) elif kind == "Tuple": temp_scope = provider.allocateTempScope("tuple_unpack") source_iter_var = provider.allocateTempVariable(temp_scope=temp_scope, name="source_iter") element_vars = [ provider.allocateTempVariable(temp_scope=temp_scope, name="element_%d" % (element_index + 1)) for element_index in range(len(detail)) ] starred_list_var = None starred_index = None statements = [] for element_index, element in enumerate(detail): if element[0] == "Starred": if starred_index is not None: raiseSyntaxError( "two starred expressions in assignment" if python_version < 390 else "multiple starred expressions in assignment", source_ref.atColumnNumber(0), ) starred_index = element_index for element_index, element in enumerate(detail): element_var = element_vars[element_index] if starred_list_var is not None: statements.insert( starred_index + 1, StatementAssignmentVariable( variable=element_var, source=ExpressionListOperationPop( list_arg=ExpressionTempVariableRef( variable=starred_list_var, source_ref=source_ref), source_ref=source_ref, ), source_ref=source_ref, ), ) elif element[0] != "Starred": statements.append( StatementAssignmentVariable( variable=element_var, source=ExpressionSpecialUnpack( value=ExpressionTempVariableRef( variable=source_iter_var, source_ref=source_ref), count=element_index + 1, expected=starred_index or len(detail), starred=starred_index is not None, source_ref=source_ref, ), source_ref=source_ref, )) else: assert starred_index == element_index starred_list_var = element_var statements.append( StatementAssignmentVariable( variable=element_var, source=ExpressionBuiltinList( value=ExpressionTempVariableRef( variable=source_iter_var, source_ref=source_ref), source_ref=source_ref, ), source_ref=source_ref, )) if starred_list_var is None: statements.append( StatementSpecialUnpackCheck( iterator=ExpressionTempVariableRef( variable=source_iter_var, source_ref=source_ref), count=len(detail), source_ref=source_ref, )) else: statements.insert( starred_index + 1, makeStatementConditional( condition=makeComparisonExpression( comparator="Lt", left=ExpressionBuiltinLen( value=ExpressionTempVariableRef( variable=starred_list_var, source_ref=source_ref), source_ref=source_ref, ), right=makeConstantRefNode( constant=len(statements) - starred_index - 1, source_ref=source_ref, ), source_ref=source_ref, ), yes_branch=makeRaiseExceptionExpressionFromTemplate( exception_type="ValueError", template="""\ not enough values to unpack (expected at least %d, got %%d)""" % (len(statements) - 1), template_args=makeBinaryOperationNode( operator="Add", left=ExpressionBuiltinLen( value=ExpressionTempVariableRef( variable=starred_list_var, source_ref=source_ref), source_ref=source_ref, ), right=makeConstantRefNode(constant=starred_index, source_ref=source_ref), source_ref=source_ref, ), source_ref=source_ref, ).asStatement(), no_branch=None, source_ref=source_ref, ), ) if python_version >= 370: iter_creation_class = ExpressionBuiltinIterForUnpack else: iter_creation_class = ExpressionBuiltinIter1 statements = [ StatementAssignmentVariable( variable=source_iter_var, source=iter_creation_class(value=source, source_ref=source_ref), source_ref=source_ref, ), makeTryFinallyStatement( provider=provider, tried=statements, final=(StatementReleaseVariable(variable=source_iter_var, source_ref=source_ref), ), source_ref=source_ref, ), ] # When all is done, copy over to the actual assignment targets, starred # or not makes no difference here anymore. for element_index, element in enumerate(detail): if element[0] == "Starred": element = element[1] element_var = element_vars[element_index] statements.append( buildAssignmentStatementsFromDecoded( provider=provider, kind=element[0], detail=element[1], source=ExpressionTempVariableRef(variable=element_var, source_ref=source_ref), source_ref=source_ref, )) # Need to release temporary variables right after successful # usage. statements.append( StatementDelVariable(variable=element_var, tolerant=True, source_ref=source_ref)) final_statements = [] for element_var in element_vars: final_statements.append( StatementReleaseVariable(variable=element_var, source_ref=source_ref)) return makeTryFinallyStatement( provider=provider, tried=statements, final=final_statements, source_ref=source_ref, ) elif kind == "Starred": raiseSyntaxError( "starred assignment target must be in a list or tuple", source_ref.atColumnNumber(0), ) else: assert False, (kind, source_ref, detail)
def buildAssignmentStatementsFromDecoded(provider, kind, detail, source, source_ref): # This is using many variable names on purpose, so as to give names to the # unpacked detail values, and has many branches due to the many cases # dealt with, pylint: disable=too-many-branches,too-many-locals if kind == "Name": variable_ref = detail return StatementAssignmentVariable( variable_ref = variable_ref, source = source, source_ref = source_ref ) elif kind == "Attribute": lookup_source, attribute_name = detail return StatementAssignmentAttribute( expression = lookup_source, attribute_name = attribute_name, source = source, source_ref = source_ref ) elif kind == "Subscript": subscribed, subscript = detail return StatementAssignmentSubscript( expression = subscribed, subscript = subscript, source = source, source_ref = source_ref ) elif kind == "Slice": lookup_source, lower, upper = detail # For Python3 there is no slicing operation, this is always done # with subscript using a slice object. For Python2, it is only done # if no "step" is provided. use_sliceobj = python_version >= 300 if use_sliceobj: return StatementAssignmentSubscript( expression = lookup_source, source = source, subscript = ExpressionBuiltinSlice( start = lower, stop = upper, step = None, source_ref = source_ref ), source_ref = source_ref ) else: return StatementAssignmentSlice( expression = lookup_source, lower = lower, upper = upper, source = source, source_ref = source_ref ) elif kind == "Tuple": temp_scope = provider.allocateTempScope("tuple_unpack") source_iter_var = provider.allocateTempVariable( temp_scope = temp_scope, name = "source_iter" ) element_vars = [ provider.allocateTempVariable( temp_scope = temp_scope, name = "element_%d" % ( element_index + 1 ) ) for element_index in range(len(detail)) ] starred_list_var = None starred_index = None statements = [] for element_index, element in enumerate(detail): element_var = element_vars[element_index] if starred_list_var is not None: if element[0] == "Starred": raiseSyntaxError( "two starred expressions in assignment", source_ref.atColumnNumber(0) ) statements.insert( starred_index+1, StatementAssignmentVariable( variable_ref = ExpressionTargetTempVariableRef( variable = element_var, source_ref = source_ref ), source = ExpressionListOperationPop( list_arg = ExpressionTempVariableRef( variable = starred_list_var, source_ref = source_ref ), source_ref = source_ref ), source_ref = source_ref ) ) elif element[0] != "Starred": statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetTempVariableRef( variable = element_var, source_ref = source_ref ), source = ExpressionSpecialUnpack( value = ExpressionTempVariableRef( variable = source_iter_var, source_ref = source_ref ), count = element_index + 1, expected = len(detail), source_ref = source_ref ), source_ref = source_ref ) ) else: starred_index = element_index starred_list_var = element_var statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetTempVariableRef( variable = element_var, source_ref = source_ref ), source = ExpressionBuiltinList( value = ExpressionTempVariableRef( variable = source_iter_var, source_ref = source_ref ), source_ref = source_ref ), source_ref = source_ref ) ) if starred_list_var is None: statements.append( StatementSpecialUnpackCheck( iterator = ExpressionTempVariableRef( variable = source_iter_var, source_ref = source_ref ), count = len(detail), source_ref = source_ref ) ) statements = [ StatementAssignmentVariable( variable_ref = ExpressionTargetTempVariableRef( variable = source_iter_var, source_ref = source_ref ), source = ExpressionBuiltinIter1( value = source, source_ref = source_ref ), source_ref = source_ref ), makeTryFinallyStatement( provider = provider, tried = statements, final = ( StatementReleaseVariable( variable = source_iter_var, source_ref = source_ref ), ), source_ref = source_ref ) ] # When all is done, copy over to the actual assignment targets, starred # or not makes no difference here anymore. for element_index, element in enumerate(detail): if element[0] == "Starred": element = element[1] element_var = element_vars[element_index] statements.append( buildAssignmentStatementsFromDecoded( provider = provider, kind = element[0], detail = element[1], source = ExpressionTempVariableRef( variable = element_var, source_ref = source_ref ), source_ref = source_ref ) ) # Need to release temporary variables right after successful # usage. statements.append( StatementDelVariable( variable_ref = ExpressionTargetTempVariableRef( variable = element_var, source_ref = source_ref ), tolerant = True, source_ref = source_ref, ) ) final_statements = [] for element_var in element_vars: final_statements.append( StatementReleaseVariable( variable = element_var, source_ref = source_ref, ) ) return makeTryFinallyStatement( provider = provider, tried = statements, final = final_statements, source_ref = source_ref ) elif kind == "Starred": raiseSyntaxError( "starred assignment target must be in a list or tuple", source_ref.atColumnNumber(0) ) else: assert False, (kind, source_ref, detail)
def buildParseTree(provider, ast_tree, source_ref, is_module, is_main): # There are a bunch of branches here, mostly to deal with version # differences for module default variables. pylint: disable=too-many-branches # Maybe one day, we do exec inlining again, that is what this is for, # then is_module won't be True, for now it always is. pushFutureSpec() if is_module: provider.setFutureSpec(getFutureSpec()) body, doc = extractDocFromBody(ast_tree) if is_module and is_main and python_version >= 0x360: provider.markAsNeedsAnnotationsDictionary() result = buildStatementsNode(provider=provider, nodes=body, source_ref=source_ref) # After building, we can verify that all future statements were where they # belong, namely at the start of the module. checkFutureImportsOnlyAtStart(body) internal_source_ref = source_ref.atInternal() statements = [] if is_module: # Add import of "site" module of main programs visibly in the node tree, # so recursion and optimization can pick it up, checking its effects. if is_main and not Options.hasPythonFlagNoSite(): statements.append( StatementExpressionOnly( expression=makeExpressionImportModuleFixed( module_name="site", source_ref=source_ref), source_ref=source_ref, )) for path_imported_name in getPthImportedPackages(): if isHardModuleWithoutSideEffect(path_imported_name): continue statements.append( StatementExpressionOnly( expression=makeExpressionImportModuleFixed( module_name=path_imported_name, source_ref=source_ref), source_ref=source_ref, )) statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__doc__", source=makeConstantRefNode(constant=doc, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__file__", source=ExpressionModuleAttributeFileRef( variable=provider.getVariableForReference("__file__"), source_ref=internal_source_ref, ), source_ref=internal_source_ref, )) if provider.isCompiledPythonPackage(): # This assigns "__path__" value. statements.append( createPathAssignment(provider, internal_source_ref)) statements.append( createImporterCacheAssignment(provider, internal_source_ref)) if python_version >= 0x340 and not is_main: statements += ( StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference("__spec__"), source_ref=internal_source_ref, ), attribute_name="origin", source=ExpressionModuleAttributeFileRef( variable=provider.getVariableForReference("__file__"), source_ref=internal_source_ref, ), source_ref=internal_source_ref, ), StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference("__spec__"), source_ref=internal_source_ref, ), attribute_name="has_location", source=makeConstantRefNode(True, internal_source_ref), source_ref=internal_source_ref, ), ) if provider.isCompiledPythonPackage(): statements.append( StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference( "__spec__"), source_ref=internal_source_ref, ), attribute_name="submodule_search_locations", source=ExpressionVariableNameRef( provider=provider, variable_name="__path__", source_ref=internal_source_ref, ), source_ref=internal_source_ref, )) if python_version >= 0x300: statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__cached__", source=ExpressionConstantNoneRef( source_ref=internal_source_ref), source_ref=internal_source_ref, )) needs__initializing__ = (not provider.isMainModule() and 0x300 <= python_version < 0x340) if needs__initializing__: # Set "__initializing__" at the beginning to True statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__initializing__", source=makeConstantRefNode(constant=True, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) if provider.needsAnnotationsDictionary(): # Set "__annotations__" on module level to {} statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__annotations__", source=makeConstantRefNode(constant={}, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) # Now the module body if there is any at all. if result is not None: statements.extend(result.subnode_statements) if needs__initializing__: # Set "__initializing__" at the end to False statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__initializing__", source=makeConstantRefNode(constant=False, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) if is_module: result = makeModuleFrame(module=provider, statements=statements, source_ref=source_ref) popFutureSpec() return result else: assert False