def buildParseTree(provider, source_code, source_ref, is_module, is_main): # There are a bunch of branches here, mostly to deal with version # differences for module default variables. pylint: disable=too-many-branches pushFutureSpec() if is_module: provider.future_spec = getFutureSpec() body = parseSourceCodeToAst(source_code=source_code, filename=source_ref.getFilename(), line_offset=source_ref.getLineNumber() - 1) body, doc = extractDocFromBody(body) if is_module and is_main and python_version >= 360: provider.markAsNeedsAnnotationsDictionary() result = buildStatementsNode(provider=provider, nodes=body, source_ref=source_ref) checkFutureImportsOnlyAtStart(body) internal_source_ref = source_ref.atInternal() statements = [] if is_module: # Add import of "site" module of main programs visibly in the node tree, # so recursion and optimization can pick it up, checking its effects. if is_main and "no_site" not in Options.getPythonFlags(): for path_imported_name in getPthImportedPackages(): statements.append( StatementExpressionOnly(expression=makeAbsoluteImportNode( module_name=path_imported_name, source_ref=source_ref, ), source_ref=source_ref)) statements.append( StatementExpressionOnly(expression=makeAbsoluteImportNode( module_name="site", source_ref=source_ref, ), source_ref=source_ref)) statements.append( StatementAssignmentVariableName(provider=provider, variable_name="__doc__", source=makeConstantRefNode( constant=doc, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__file__", source=ExpressionModuleAttributeFileRef( module=provider, source_ref=internal_source_ref, ), source_ref=internal_source_ref)) if provider.isCompiledPythonPackage(): # This assigns "__path__" value. statements.append( createPathAssignment(provider, internal_source_ref)) if python_version >= 300: statements.append( StatementAssignmentVariableName(provider=provider, variable_name="__cached__", source=ExpressionConstantNoneRef( source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) needs__initializing__ = not provider.isMainModule( ) and 300 <= python_version < 340 if needs__initializing__: # Set "__initializing__" at the beginning to True statements.append( StatementAssignmentVariableName(provider=provider, variable_name="__initializing__", source=makeConstantRefNode( constant=True, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) if provider.needsAnnotationsDictionary(): # Set "__annotations__" on module level to {} statements.append( StatementAssignmentVariableName(provider=provider, variable_name="__annotations__", source=makeConstantRefNode( constant={}, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) # Now the module body if there is any at all. if result is not None: statements.extend(result.getStatements()) if needs__initializing__: # Set "__initializing__" at the end to False statements.append( StatementAssignmentVariableName(provider=provider, variable_name="__initializing__", source=makeConstantRefNode( constant=False, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) if is_module: result = makeModuleFrame(module=provider, statements=statements, source_ref=source_ref) popFutureSpec() return result else: assert False
def buildParseTree(provider, source_code, source_ref, is_module, is_main): # There are a bunch of branches here, mostly to deal with version # differences for module default variables. pylint: disable=R0912 # Workaround: ast.parse cannot cope with some situations where a file is not # terminated by a new line. if not source_code.endswith('\n'): source_code = source_code + '\n' try: body = ast.parse(source_code, source_ref.getFilename()) except SyntaxError as e: _makeSyntaxErrorCompatible(e) raise e assert getKind(body) == "Module" line_offset = source_ref.getLineNumber() - 1 if line_offset > 0: for created_node in ast.walk(body): if hasattr(created_node, "lineno"): created_node.lineno += line_offset body, doc = extractDocFromBody(body) result = buildStatementsNode(provider=provider, nodes=body, source_ref=source_ref) checkFutureImportsOnlyAtStart(body) internal_source_ref = source_ref.atInternal() statements = [] if is_module: # Add import of "site" module of main programs visibly in the node tree, # so recursion and optimization can pick it up, checking its effects. if is_main and "no_site" not in Options.getPythonFlags(): for path_imported_name in getPthImportedPackages(): statements.append( StatementExpressionOnly(expression=ExpressionImportModule( module_name=path_imported_name, import_list=(), level=0, source_ref=source_ref, ), source_ref=source_ref)) statements.append( StatementExpressionOnly(expression=ExpressionImportModule( module_name="site", import_list=(), level=0, source_ref=source_ref, ), source_ref=source_ref)) statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__doc__", source_ref=internal_source_ref), source=ExpressionConstantRef(constant=doc, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__file__", source_ref=internal_source_ref), source=ExpressionModuleFileAttributeRef( source_ref=internal_source_ref, ), source_ref=internal_source_ref)) if provider.isCompiledPythonPackage(): # This assigns "__path__" value. statements.append(createPathAssignment(internal_source_ref)) if python_version >= 300: statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__cached__", source_ref=internal_source_ref), source=ExpressionConstantRef(constant=None, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) if python_version >= 330: # For Python3.3, it's set for both packages and non-packages. statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__package__", source_ref=internal_source_ref), source=ExpressionConstantRef( constant=provider.getFullName() if provider.isCompiledPythonPackage() else provider.getPackage(), source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) needs__initializing__ = not provider.isMainModule() and \ (python_version >= 330 and python_version < 340) if needs__initializing__: # Set "__initializing__" at the beginning to True statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__initializing__", source_ref=internal_source_ref), source=ExpressionConstantRef(constant=True, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) # Now the module body if there is any at all. if result is not None: statements.extend(result.getStatements()) if needs__initializing__: # Set "__initializing__" at the end to False statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__initializing__", source_ref=internal_source_ref), source=ExpressionConstantRef(constant=False, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref)) if is_module: return makeModuleFrame(module=provider, statements=statements, source_ref=source_ref) else: assert False
def buildParseTree(provider, ast_tree, source_ref, is_module, is_main): # There are a bunch of branches here, mostly to deal with version # differences for module default variables. pylint: disable=too-many-branches # Maybe one day, we do exec inlining again, that is what this is for, # then is_module won't be True, for now it always is. pushFutureSpec() if is_module: provider.setFutureSpec(getFutureSpec()) body, doc = extractDocFromBody(ast_tree) if is_module and is_main and python_version >= 0x360: provider.markAsNeedsAnnotationsDictionary() result = buildStatementsNode(provider=provider, nodes=body, source_ref=source_ref) # After building, we can verify that all future statements were where they # belong, namely at the start of the module. checkFutureImportsOnlyAtStart(body) internal_source_ref = source_ref.atInternal() statements = [] if is_module: # Add import of "site" module of main programs visibly in the node tree, # so recursion and optimization can pick it up, checking its effects. if is_main and not Options.hasPythonFlagNoSite(): statements.append( StatementExpressionOnly( expression=makeExpressionImportModuleFixed( module_name="site", source_ref=source_ref), source_ref=source_ref, )) for path_imported_name in getPthImportedPackages(): if isHardModuleWithoutSideEffect(path_imported_name): continue statements.append( StatementExpressionOnly( expression=makeExpressionImportModuleFixed( module_name=path_imported_name, source_ref=source_ref), source_ref=source_ref, )) statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__doc__", source=makeConstantRefNode(constant=doc, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__file__", source=ExpressionModuleAttributeFileRef( variable=provider.getVariableForReference("__file__"), source_ref=internal_source_ref, ), source_ref=internal_source_ref, )) if provider.isCompiledPythonPackage(): # This assigns "__path__" value. statements.append( createPathAssignment(provider, internal_source_ref)) statements.append( createImporterCacheAssignment(provider, internal_source_ref)) if python_version >= 0x340 and not is_main: statements += ( StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference("__spec__"), source_ref=internal_source_ref, ), attribute_name="origin", source=ExpressionModuleAttributeFileRef( variable=provider.getVariableForReference("__file__"), source_ref=internal_source_ref, ), source_ref=internal_source_ref, ), StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference("__spec__"), source_ref=internal_source_ref, ), attribute_name="has_location", source=makeConstantRefNode(True, internal_source_ref), source_ref=internal_source_ref, ), ) if provider.isCompiledPythonPackage(): statements.append( StatementAssignmentAttribute( expression=ExpressionModuleAttributeSpecRef( variable=provider.getVariableForReference( "__spec__"), source_ref=internal_source_ref, ), attribute_name="submodule_search_locations", source=ExpressionVariableNameRef( provider=provider, variable_name="__path__", source_ref=internal_source_ref, ), source_ref=internal_source_ref, )) if python_version >= 0x300: statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__cached__", source=ExpressionConstantNoneRef( source_ref=internal_source_ref), source_ref=internal_source_ref, )) needs__initializing__ = (not provider.isMainModule() and 0x300 <= python_version < 0x340) if needs__initializing__: # Set "__initializing__" at the beginning to True statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__initializing__", source=makeConstantRefNode(constant=True, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) if provider.needsAnnotationsDictionary(): # Set "__annotations__" on module level to {} statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__annotations__", source=makeConstantRefNode(constant={}, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) # Now the module body if there is any at all. if result is not None: statements.extend(result.subnode_statements) if needs__initializing__: # Set "__initializing__" at the end to False statements.append( StatementAssignmentVariableName( provider=provider, variable_name="__initializing__", source=makeConstantRefNode(constant=False, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, )) if is_module: result = makeModuleFrame(module=provider, statements=statements, source_ref=source_ref) popFutureSpec() return result else: assert False
def buildParseTree(provider, source_code, source_ref, is_module, is_main): # There are a bunch of branches here, mostly to deal with version # differences for module default variables. body = parseSourceCodeToAst( source_code = source_code, filename = source_ref.getFilename(), line_offset = source_ref.getLineNumber() - 1 ) body, doc = extractDocFromBody(body) result = buildStatementsNode( provider = provider, nodes = body, source_ref = source_ref ) checkFutureImportsOnlyAtStart(body) internal_source_ref = source_ref.atInternal() statements = [] if is_module: # Add import of "site" module of main programs visibly in the node tree, # so recursion and optimization can pick it up, checking its effects. if is_main and "no_site" not in Options.getPythonFlags(): for path_imported_name in getPthImportedPackages(): statements.append( StatementExpressionOnly( expression = ExpressionImportModule( module_name = path_imported_name, import_list = (), level = 0, source_ref = source_ref, ), source_ref = source_ref ) ) statements.append( StatementExpressionOnly( expression = ExpressionImportModule( module_name = "site", import_list = (), level = 0, source_ref = source_ref, ), source_ref = source_ref ) ) statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetVariableRef( variable_name = "__doc__", source_ref = internal_source_ref ), source = ExpressionConstantRef( constant = doc, source_ref = internal_source_ref, user_provided = True ), source_ref = internal_source_ref ) ) statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetVariableRef( variable_name = "__file__", source_ref = internal_source_ref ), source = ExpressionModuleFileAttributeRef( source_ref = internal_source_ref, ), source_ref = internal_source_ref ) ) if provider.isCompiledPythonPackage(): # This assigns "__path__" value. statements.append( createPathAssignment(internal_source_ref) ) if python_version >= 300: statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetVariableRef( variable_name = "__cached__", source_ref = internal_source_ref ), source = ExpressionConstantRef( constant = None, source_ref = internal_source_ref, user_provided = True ), source_ref = internal_source_ref ) ) if python_version >= 330: # For Python3.3, it's set for both packages and non-packages. statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetVariableRef( variable_name = "__package__", source_ref = internal_source_ref ), source = ExpressionConstantRef( constant = provider.getFullName() if provider.isCompiledPythonPackage() else provider.getPackage(), source_ref = internal_source_ref, user_provided = True ), source_ref = internal_source_ref ) ) needs__initializing__ = not provider.isMainModule() and \ (python_version >= 330 and python_version < 340) if needs__initializing__: # Set "__initializing__" at the beginning to True statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetVariableRef( variable_name = "__initializing__", source_ref = internal_source_ref ), source = ExpressionConstantRef( constant = True, source_ref = internal_source_ref, user_provided = True ), source_ref = internal_source_ref ) ) # Now the module body if there is any at all. if result is not None: statements.extend( result.getStatements() ) if needs__initializing__: # Set "__initializing__" at the end to False statements.append( StatementAssignmentVariable( variable_ref = ExpressionTargetVariableRef( variable_name = "__initializing__", source_ref = internal_source_ref ), source = ExpressionConstantRef( constant = False, source_ref = internal_source_ref, user_provided = True ), source_ref = internal_source_ref ) ) if is_module: return makeModuleFrame( module = provider, statements = statements, source_ref = source_ref ) else: assert False
def buildParseTree(provider, source_code, source_ref, is_module, is_main): # There are a bunch of branches here, mostly to deal with version # differences for module default variables. pylint: disable=R0912 # Workaround: ast.parse cannot cope with some situations where a file is not # terminated by a new line. if not source_code.endswith("\n"): source_code = source_code + "\n" try: body = ast.parse(source_code, source_ref.getFilename()) except SyntaxError as e: _makeSyntaxErrorCompatible(e) raise e assert getKind(body) == "Module" line_offset = source_ref.getLineNumber() - 1 if line_offset > 0: for created_node in ast.walk(body): if hasattr(created_node, "lineno"): created_node.lineno += line_offset body, doc = extractDocFromBody(body) result = buildStatementsNode(provider=provider, nodes=body, source_ref=source_ref) checkFutureImportsOnlyAtStart(body) internal_source_ref = source_ref.atInternal() statements = [] if is_module: # Add import of "site" module of main programs visibly in the node tree, # so recursion and optimization can pick it up, checking its effects. if is_main and "no_site" not in Options.getPythonFlags(): for path_imported_name in getPthImportedPackages(): statements.append( StatementExpressionOnly( expression=ExpressionImportModule( module_name=path_imported_name, import_list=(), level=0, source_ref=source_ref ), source_ref=source_ref, ) ) statements.append( StatementExpressionOnly( expression=ExpressionImportModule( module_name="site", import_list=(), level=0, source_ref=source_ref ), source_ref=source_ref, ) ) statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef(variable_name="__doc__", source_ref=internal_source_ref), source=ExpressionConstantRef(constant=doc, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, ) ) statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef(variable_name="__file__", source_ref=internal_source_ref), source=ExpressionModuleFileAttributeRef(source_ref=internal_source_ref), source_ref=internal_source_ref, ) ) if provider.isCompiledPythonPackage(): # This assigns "__path__" value. statements.append(createPathAssignment(internal_source_ref)) if python_version >= 300: statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef(variable_name="__cached__", source_ref=internal_source_ref), source=ExpressionConstantRef(constant=None, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, ) ) if python_version >= 330: # For Python3.3, it's set for both packages and non-packages. statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef(variable_name="__package__", source_ref=internal_source_ref), source=ExpressionConstantRef( constant=provider.getFullName() if provider.isCompiledPythonPackage() else provider.getPackage(), source_ref=internal_source_ref, user_provided=True, ), source_ref=internal_source_ref, ) ) needs__initializing__ = not provider.isMainModule() and (python_version >= 330 and python_version < 340) if needs__initializing__: # Set "__initializing__" at the beginning to True statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__initializing__", source_ref=internal_source_ref ), source=ExpressionConstantRef(constant=True, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, ) ) # Now the module body if there is any at all. if result is not None: statements.extend(result.getStatements()) if needs__initializing__: # Set "__initializing__" at the end to False statements.append( StatementAssignmentVariable( variable_ref=ExpressionTargetVariableRef( variable_name="__initializing__", source_ref=internal_source_ref ), source=ExpressionConstantRef(constant=False, source_ref=internal_source_ref, user_provided=True), source_ref=internal_source_ref, ) ) if is_module: return makeModuleFrame(module=provider, statements=statements, source_ref=source_ref) else: assert False