def add_new_class_for_module(module: MypyFile, name: str, bases: List[Instance], fields: 'OrderedDict[str, MypyType]') -> TypeInfo: new_class_unique_name = checker.gen_unique_name(name, module.names) # make new class expression classdef = ClassDef(new_class_unique_name, Block([])) classdef.fullname = module.fullname() + '.' + new_class_unique_name # make new TypeInfo new_typeinfo = TypeInfo(SymbolTable(), classdef, module.fullname()) new_typeinfo.bases = bases calculate_mro(new_typeinfo) new_typeinfo.calculate_metaclass_type() def add_field_to_new_typeinfo(var: Var, is_initialized_in_class: bool = False, is_property: bool = False) -> None: var.info = new_typeinfo var.is_initialized_in_class = is_initialized_in_class var.is_property = is_property var._fullname = new_typeinfo.fullname() + '.' + var.name() new_typeinfo.names[var.name()] = SymbolTableNode(MDEF, var) # add fields var_items = [Var(item, typ) for item, typ in fields.items()] for var_item in var_items: add_field_to_new_typeinfo(var_item, is_property=True) classdef.info = new_typeinfo module.names[new_class_unique_name] = SymbolTableNode( GDEF, new_typeinfo, plugin_generated=True) return new_typeinfo
def anal_defs(self, defs, fnam, mod_id): """Perform the first analysis pass. Resolve the full names of definitions and construct type info structures, but do not resolve inter-definition references such as base classes. """ self.cur_mod_id = mod_id self.errors.set_file(fnam) self.globals = SymbolTable() self.global_decls = [set()] # Add implicit definition of '__name__'. name_def = VarDef([Var("__name__", Any())], True) defs.insert(0, name_def) for d in defs: if isinstance(d, AssignmentStmt): self.anal_assignment_stmt(d) elif isinstance(d, FuncDef): self.anal_func_def(d) elif isinstance(d, OverloadedFuncDef): self.anal_overloaded_func_def(d) elif isinstance(d, TypeDef): self.anal_type_def(d) elif isinstance(d, VarDef): self.anal_var_def(d) elif isinstance(d, ForStmt): self.anal_for_stmt(d) # Add implicit definition of 'None' to builtins, as we cannot define a # variable with a None type explicitly. if mod_id == "builtins": none_def = VarDef([Var("None", NoneTyp())], True) defs.append(none_def) self.anal_var_def(none_def)
def process(names: SymbolTable, is_stub_file: bool, prefix: str, errors: Errors) -> None: for name, symnode in names.items(): node = symnode.node if isinstance(node, TypeInfo) and node.fullname().startswith(prefix): calculate_class_abstract_status(node, is_stub_file, errors) new_prefix = prefix + '.' + node.name() process(node.names, is_stub_file, new_prefix, errors)
def visit_symbol_table(self, symtab: SymbolTable) -> None: # Copy the items because we may mutate symtab. for key, value in list(symtab.items()): cross_ref = value.cross_ref if cross_ref is not None: # Fix up cross-reference. del value.cross_ref if cross_ref in self.modules: value.node = self.modules[cross_ref] else: stnode = lookup_qualified_stnode(self.modules, cross_ref, self.quick_and_dirty) if stnode is not None: value.node = stnode.node value.type_override = stnode.type_override elif not self.quick_and_dirty: assert stnode is not None, "Could not find cross-ref %s" % ( cross_ref, ) else: if isinstance(value.node, TypeInfo): # TypeInfo has no accept(). TODO: Add it? self.visit_type_info(value.node) elif value.node is not None: value.node.accept(self) if value.type_override is not None: value.type_override.accept(self.type_fixer)
def dump_typeinfos_recursive(self, names: SymbolTable) -> List[str]: a = [] for name, node in sorted(names.items(), key=lambda x: x[0]): if isinstance(node.node, TypeInfo): a.extend(self.dump_typeinfo(node.node)) a.extend(self.dump_typeinfos_recursive(node.node.names)) return a
def _dynamic_class_hook(ctx: DynamicClassDefContext) -> None: """Generate a declarative Base class when the declarative_base() function is encountered.""" _add_globals(ctx) cls = ClassDef(ctx.name, Block([])) cls.fullname = ctx.api.qualified_name(ctx.name) info = TypeInfo(SymbolTable(), cls, ctx.api.cur_mod_id) cls.info = info _set_declarative_metaclass(ctx.api, cls) cls_arg = util.get_callexpr_kwarg(ctx.call, "cls", expr_types=(NameExpr, )) if cls_arg is not None and isinstance(cls_arg.node, TypeInfo): util.set_is_base(cls_arg.node) decl_class.scan_declarative_assignments_and_apply_types( cls_arg.node.defn, ctx.api, is_mixin_scan=True) info.bases = [Instance(cls_arg.node, [])] else: obj = ctx.api.named_type(names.NAMED_TYPE_BUILTINS_OBJECT) info.bases = [obj] try: calculate_mro(info) except MroError: util.fail(ctx.api, "Not able to calculate MRO for declarative base", ctx.call) obj = ctx.api.named_type(names.NAMED_TYPE_BUILTINS_OBJECT) info.bases = [obj] info.fallback_to_any = True ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) util.set_is_base(info)
def build_class_with_annotated_fields(api: 'TypeChecker', base: Type, fields: 'OrderedDict[str, Type]', name: str) -> Instance: """Build an Instance with `name` that contains the specified `fields` as attributes and extends `base`.""" # Credit: This code is largely copied/modified from TypeChecker.intersect_instance_callable and # NamedTupleAnalyzer.build_namedtuple_typeinfo from mypy.checker import gen_unique_name cur_module = cast(MypyFile, api.scope.stack[0]) gen_name = gen_unique_name(name, cur_module.names) cdef = ClassDef(name, Block([])) cdef.fullname = cur_module.fullname() + '.' + gen_name info = TypeInfo(SymbolTable(), cdef, cur_module.fullname()) cdef.info = info info.bases = [base] def add_field(var: Var, is_initialized_in_class: bool = False, is_property: bool = False) -> None: var.info = info var.is_initialized_in_class = is_initialized_in_class var.is_property = is_property var._fullname = '%s.%s' % (info.fullname(), var.name()) info.names[var.name()] = SymbolTableNode(MDEF, var) vars = [Var(item, typ) for item, typ in fields.items()] for var in vars: add_field(var, is_property=True) calculate_mro(info) info.calculate_metaclass_type() cur_module.names[gen_name] = SymbolTableNode(GDEF, info, plugin_generated=True) return Instance(info, [])
def query_callback(ctx: DynamicClassDefContext) -> TypeInfo: # todo be defensive--ctx.type is Schema[Literal[fname]] #fname = ctx.arg_types[0].value query = ctx.call.args[1].value defn = ClassDef( ctx.name, defs=Block([ mpn.AssignmentStmt( lvalues=mpn.NameExpr, rvalue=None, type=ctx.api.lookup_fully_qualified_or_none('builtins.str'), new_syntax=True) ])) defn.fullname = ctx.api.qualified_name(ctx.name) names = SymbolTable() var = Var('me', ctx.api.builtin_type('builtins.str')) var.info = var.type.type var.is_property = True names['me'] = SymbolTableNode(MDEF, var, plugin_generated=True) info = TypeInfo(names=names, defn=defn, module_name=ctx.api.cur_mod_id) obj = ctx.api.builtin_type('builtins.object') info.mro = [info, obj.type] info.bases = [obj] print(ctx.name, info) ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info))
def compute_all_mros(symtab: SymbolTable, modules: Dict[str, MypyFile]) -> None: for key, value in symtab.items(): if value.kind in (LDEF, MDEF, GDEF) and isinstance(value.node, TypeInfo): info = value.node info.calculate_mro() assert info.mro, "No MRO calculated for %s" % (info.fullname(),) compute_all_mros(info.names, modules)
def make_type_info(name: str, is_abstract: bool = False, mro: List[TypeInfo] = None, bases: List[Instance] = None, typevars: List[str] = None) -> TypeInfo: """Make a TypeInfo suitable for use in unit tests.""" type_def = TypeDef(name, Block([]), None, []) type_def.fullname = name if typevars: v = [] # type: List[TypeVarDef] id = 1 for n in typevars: v.append(TypeVarDef(n, id, None)) id += 1 type_def.type_vars = v info = TypeInfo(SymbolTable(), type_def) if mro is None: mro = [] info.mro = [info] + mro if bases is None: if mro: # By default, assume that there is a single non-generic base. bases = [Instance(mro[0], [])] else: bases = [] info.bases = bases return info
def _dynamic_class_hook(ctx: DynamicClassDefContext) -> None: """Generate a declarative Base class when the declarative_base() function is encountered.""" cls = ClassDef(ctx.name, Block([])) cls.fullname = ctx.api.qualified_name(ctx.name) info = TypeInfo(SymbolTable(), cls, ctx.api.cur_mod_id) cls.info = info _make_declarative_meta(ctx.api, cls) cls_arg = util._get_callexpr_kwarg(ctx.call, "cls") if cls_arg is not None: decl_class._scan_declarative_assignments_and_apply_types( cls_arg.node.defn, ctx.api, is_mixin_scan=True) info.bases = [Instance(cls_arg.node, [])] else: obj = ctx.api.builtin_type("builtins.object") info.bases = [obj] try: calculate_mro(info) except MroError: util.fail(ctx.api, "Not able to calculate MRO for declarative base", ctx.call) obj = ctx.api.builtin_type("builtins.object") info.bases = [obj] info.fallback_to_any = True ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info))
def visit_symbol_table(self, symtab: SymbolTable) -> None: # Copy the items because we may mutate symtab. for key, value in list(symtab.items()): cross_ref = value.cross_ref if cross_ref is not None: # Fix up cross-reference. value.cross_ref = None if cross_ref in self.modules: value.node = self.modules[cross_ref] else: stnode = lookup_qualified_stnode(self.modules, cross_ref, self.allow_missing) if stnode is not None: value.node = stnode.node elif not self.allow_missing: assert stnode is not None, "Could not find cross-ref %s" % ( cross_ref, ) else: # We have a missing crossref in allow missing mode, need to put something value.node = missing_info(self.modules) else: if isinstance(value.node, TypeInfo): # TypeInfo has no accept(). TODO: Add it? self.visit_type_info(value.node) elif value.node is not None: value.node.accept(self) else: assert False, 'Unexpected empty node %r: %s' % (key, value)
def replacement_map_from_symbol_table( old: SymbolTable, new: SymbolTable, prefix: str) -> Dict[SymbolNode, SymbolNode]: """Create a new-to-old object identity map by comparing two symbol table revisions. Both symbol tables must refer to revisions of the same module id. The symbol tables are compared recursively (recursing into nested class symbol tables), but only within the given module prefix. Don't recurse into other modules accessible through the symbol table. """ replacements = {} # type: Dict[SymbolNode, SymbolNode] for name, node in old.items(): if (name in new and (node.kind == MDEF or node.node and get_prefix(node.node.fullname()) == prefix)): new_node = new[name] if (type(new_node.node) == type(node.node) # noqa and new_node.node and node.node and new_node.node.fullname() == node.node.fullname() and new_node.kind == node.kind): replacements[new_node.node] = node.node if isinstance(node.node, TypeInfo) and isinstance(new_node.node, TypeInfo): type_repl = replacement_map_from_symbol_table( node.node.names, new_node.node.names, prefix) replacements.update(type_repl) return replacements
def visit_symbol_table(self, symtab: SymbolTable, table_fullname: str) -> None: # Copy the items because we may mutate symtab. for key, value in list(symtab.items()): cross_ref = value.cross_ref if cross_ref is not None: # Fix up cross-reference. value.cross_ref = None if cross_ref in self.modules: value.node = self.modules[cross_ref] else: stnode = lookup_fully_qualified( cross_ref, self.modules, raise_on_missing=not self.allow_missing) if stnode is not None: assert stnode.node is not None, (table_fullname + "." + key, cross_ref) value.node = stnode.node elif not self.allow_missing: assert False, f"Could not find cross-ref {cross_ref}" else: # We have a missing crossref in allow missing mode, need to put something value.node = missing_info(self.modules) else: if isinstance(value.node, TypeInfo): # TypeInfo has no accept(). TODO: Add it? self.visit_type_info(value.node) elif value.node is not None: value.node.accept(self) else: assert False, f'Unexpected empty node {key!r}: {value}'
def add_type_promotion(info: TypeInfo, module_names: SymbolTable, options: Options) -> None: """Setup extra, ad-hoc subtyping relationships between classes (promotion). This includes things like 'int' being compatible with 'float'. """ defn = info.defn promote_target = None # type: Optional[Type] for decorator in defn.decorators: if isinstance(decorator, CallExpr): analyzed = decorator.analyzed if isinstance(analyzed, PromoteExpr): # _promote class decorator (undocumented feature). promote_target = analyzed.type if not promote_target: promotions = (TYPE_PROMOTIONS_PYTHON3 if options.python_version[0] >= 3 else TYPE_PROMOTIONS_PYTHON2) if defn.fullname in promotions: target_sym = module_names.get(promotions[defn.fullname]) # With test stubs, the target may not exist. if target_sym: target_info = target_sym.node assert isinstance(target_info, TypeInfo) promote_target = Instance(target_info, []) defn.info._promote = promote_target
def save_namedtuple_body(self, named_tuple_info: TypeInfo) -> Iterator[None]: """Preserve the generated body of class-based named tuple and then restore it. Temporarily clear the names dict so we don't get errors about duplicate names that were already set in build_namedtuple_typeinfo (we already added the tuple field names while generating the TypeInfo, and actual duplicates are already reported). """ nt_names = named_tuple_info.names named_tuple_info.names = SymbolTable() yield # Make sure we didn't use illegal names, then reset the names in the typeinfo. for prohibited in NAMEDTUPLE_PROHIBITED_NAMES: if prohibited in named_tuple_info.names: if nt_names.get( prohibited) is named_tuple_info.names[prohibited]: continue ctx = named_tuple_info.names[prohibited].node assert ctx is not None self.fail( 'Cannot overwrite NamedTuple attribute "{}"'.format( prohibited), ctx) # Restore the names in the original symbol table. This ensures that the symbol # table contains the field objects created by build_namedtuple_typeinfo. Exclude # __doc__, which can legally be overwritten by the class. named_tuple_info.names.update({ key: value for key, value in nt_names.items() if key not in named_tuple_info.names or key != '__doc__' })
def add_new_class_for_module( module: MypyFile, name: str, bases: List[Instance], fields: Optional[Dict[str, MypyType]] = None ) -> TypeInfo: new_class_unique_name = checker.gen_unique_name(name, module.names) # make new class expression classdef = ClassDef(new_class_unique_name, Block([])) classdef.fullname = module.fullname + "." + new_class_unique_name # make new TypeInfo new_typeinfo = TypeInfo(SymbolTable(), classdef, module.fullname) new_typeinfo.bases = bases calculate_mro(new_typeinfo) new_typeinfo.calculate_metaclass_type() # add fields if fields: for field_name, field_type in fields.items(): var = Var(field_name, type=field_type) var.info = new_typeinfo var._fullname = new_typeinfo.fullname + "." + field_name new_typeinfo.names[field_name] = SymbolTableNode(MDEF, var, plugin_generated=True) classdef.info = new_typeinfo module.names[new_class_unique_name] = SymbolTableNode(GDEF, new_typeinfo, plugin_generated=True) return new_typeinfo
def visit_symbol_table(self, symtab: SymbolTable) -> None: # Copy the items because we may mutate symtab. for key, value in list(symtab.items()): cross_ref = value.cross_ref if cross_ref is not None: # Fix up cross-reference. del value.cross_ref if cross_ref in self.modules: value.node = self.modules[cross_ref] else: stnode = lookup_qualified_stnode(self.modules, cross_ref, self.quick_and_dirty) if stnode is not None: value.node = stnode.node value.type_override = stnode.type_override if (self.quick_and_dirty and value.kind == TYPE_ALIAS and stnode.type_override is None): value.type_override = Instance(stale_info(self.modules), []) value.alias_tvars = stnode.alias_tvars or [] elif not self.quick_and_dirty: assert stnode is not None, "Could not find cross-ref %s" % (cross_ref,) else: # We have a missing crossref in quick mode, need to put something value.node = stale_info(self.modules) if value.kind == TYPE_ALIAS: value.type_override = Instance(stale_info(self.modules), []) else: if isinstance(value.node, TypeInfo): # TypeInfo has no accept(). TODO: Add it? self.visit_type_info(value.node) elif value.node is not None: value.node.accept(self) if value.type_override is not None: value.type_override.accept(self.type_fixer)
def visit_symbol_table(self, symtab: SymbolTable) -> None: # Copy the items because we may mutate symtab. for key, value in list(symtab.items()): cross_ref = value.cross_ref if cross_ref is not None: # Fix up cross-reference. del value.cross_ref if cross_ref in self.modules: value.node = self.modules[cross_ref] else: stnode = lookup_qualified_stnode(self.modules, cross_ref, self.quick_and_dirty) if stnode is not None: value.node = stnode.node value.type_override = stnode.type_override if (self.quick_and_dirty and value.kind == TYPE_ALIAS and stnode.type_override is None): value.type_override = Instance(stale_info(), []) value.alias_tvars = stnode.alias_tvars or [] elif not self.quick_and_dirty: assert stnode is not None, "Could not find cross-ref %s" % ( cross_ref, ) else: # We have a missing crossref in quick mode, need to put something value.node = stale_info() if value.kind == TYPE_ALIAS: value.type_override = Instance(stale_info(), []) else: if isinstance(value.node, TypeInfo): # TypeInfo has no accept(). TODO: Add it? self.visit_type_info(value.node) elif value.node is not None: value.node.accept(self) if value.type_override is not None: value.type_override.accept(self.type_fixer)
def visit_mypy_file(self, node: MypyFile) -> Node: # NOTE: The 'names' and 'imports' instance variables will be empty! new = MypyFile(self.nodes(node.defs), [], node.is_bom) new._name = node._name new._fullname = node._fullname new.path = node.path new.names = SymbolTable() return new
def stale_info() -> TypeInfo: suggestion = "<stale cache: consider running mypy without --quick>" dummy_def = ClassDef(suggestion, Block([])) dummy_def.fullname = suggestion info = TypeInfo(SymbolTable(), dummy_def, "<stale>") info.mro = [info] info.bases = [] return info
def visit_mypy_file(self, node: MypyFile) -> MypyFile: # NOTE: The 'names' and 'imports' instance variables will be empty! new = MypyFile(self.statements(node.defs), [], node.is_bom, ignored_lines=set(node.ignored_lines)) new._name = node._name new._fullname = node._fullname new.path = node.path new.names = SymbolTable() return new
def missing_info(modules: Dict[str, MypyFile]) -> TypeInfo: suggestion = _SUGGESTION.format('info') dummy_def = ClassDef(suggestion, Block([])) dummy_def.fullname = suggestion info = TypeInfo(SymbolTable(), dummy_def, "<missing>") obj_type = lookup_fully_qualified_typeinfo(modules, 'builtins.object', allow_missing=False) info.bases = [Instance(obj_type, [])] info.mro = [info, obj_type] return info
def replace_nodes_in_symbol_table(symbols: SymbolTable, replacements: Dict[SymbolNode, SymbolNode]) -> None: for name, node in symbols.items(): if node.node in replacements: new = replacements[node.node] new.__dict__ = node.node.__dict__ node.node = new if isinstance(node.node, Var) and node.node.type: node.node.type.accept(TypeReplaceVisitor(replacements)) node.node.info = cast(TypeInfo, replacements.get(node.node.info, node.node.info))
def visit_mypy_file(self, node: MypyFile) -> MypyFile: # NOTE: The 'names' and 'imports' instance variables will be empty! ignored_lines = {line: codes[:] for line, codes in node.ignored_lines.items()} new = MypyFile(self.statements(node.defs), [], node.is_bom, ignored_lines=ignored_lines) new._fullname = node._fullname new.path = node.path new.names = SymbolTable() return new
def add_info_hook(ctx) -> None: class_def = ClassDef(ctx.name, Block([])) class_def.fullname = ctx.api.qualified_name(ctx.name) info = TypeInfo(SymbolTable(), class_def, ctx.api.cur_mod_id) class_def.info = info obj = ctx.api.named_type('builtins.object') info.mro = [info, obj.type] info.bases = [obj] ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) info.metadata['magic'] = True
def add_info_hook(ctx): class_def = ClassDef(ctx.name, Block([])) class_def.fullname = ctx.api.qualified_name(ctx.name) info = TypeInfo(SymbolTable(), class_def, ctx.api.cur_mod_id) class_def.info = info obj = ctx.api.builtin_type('builtins.object') info.mro = [info, obj.type] info.bases = [obj] ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) DECL_BASES.add(class_def.fullname)
def stale_info(modules: Dict[str, MypyFile]) -> TypeInfo: suggestion = "<stale cache: consider running mypy without --quick>" dummy_def = ClassDef(suggestion, Block([])) dummy_def.fullname = suggestion info = TypeInfo(SymbolTable(), dummy_def, "<stale>") obj_type = lookup_qualified(modules, 'builtins.object', False) assert isinstance(obj_type, TypeInfo) info.bases = [Instance(obj_type, [])] info.mro = [info, obj_type] return info
def named_hook(ctx: DynamicClassDefContext) -> None: breakpoint() info = TypeInfo(SymbolTable(), ctx.call.args[1], ctx.api.cur_mod_id) ctx.call.args[1].info = info obj = ctx.api.builtin_type("builtins.object") info.mro = [info, obj.type] info.bases = [obj] ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(GDEF, info)) print("hoi") return
def missing_info(modules: Dict[str, MypyFile]) -> TypeInfo: suggestion = "<missing info: *should* have gone away during fine-grained update>" dummy_def = ClassDef(suggestion, Block([])) dummy_def.fullname = suggestion info = TypeInfo(SymbolTable(), dummy_def, "<missing>") obj_type = lookup_qualified(modules, 'builtins.object', False) assert isinstance(obj_type, TypeInfo) info.bases = [Instance(obj_type, [])] info.mro = [info, obj_type] return info
def apply_interface( iface_arg: Expression, class_info: TypeInfo, api: SemanticAnalyzerPluginInterface, context: Context, ) -> None: if not isinstance(iface_arg, RefExpr): api.fail("Argument to implementer should be a ref expression", iface_arg) return iface_name = iface_arg.fullname if iface_name is None: # unknown interface, probably from stubless package return iface_type = iface_arg.node if iface_type is None: return if not isinstance(iface_type, TypeInfo): # Possibly an interface from unimported package, ignore return if not self._is_interface(iface_type): api.fail( f"zope.interface.implementer accepts interface, " f"not {iface_name}.", iface_arg, ) api.fail( f"Make sure you have stubs for all packages that " f"provide interfaces for {iface_name} class hierarchy.", iface_arg, ) return # print("CLASS INFO", class_info) md = self._get_metadata(class_info) if "implements" not in md: md["implements"] = [] # impl_list = cast(List[str], md['implements']) md["implements"].append(iface_type.fullname) self.log(f"Found implementation of " f"{iface_type.fullname}: {class_info.fullname}") # Make sure implementation is treated as a subtype of an interface. Pretend # there is a decorator for the class that will create a "type promotion", # but ensure this only gets applied a single time per interface. promote = Instance(iface_type, []) if not any(ti._promote == promote for ti in class_info.mro): faketi = TypeInfo(SymbolTable(), iface_type.defn, iface_type.module_name) faketi._promote = promote class_info.mro.append(faketi)
def replace_nodes_in_symbol_table(symbols: SymbolTable, replacements: Dict[SymbolNode, SymbolNode]) -> None: for name, node in symbols.items(): if node.node: if node.node in replacements: new = replacements[node.node] old = node.node replace_object_state(new, old) node.node = new if isinstance(node.node, (Var, TypeAlias)): # Handle them here just in case these aren't exposed through the AST. node.node.accept(NodeReplaceVisitor(replacements))
def replace_nodes_in_symbol_table(symbols: SymbolTable, replacements: Dict[SymbolNode, SymbolNode]) -> None: for name, node in symbols.items(): if node.node: if node.node in replacements: new = replacements[node.node] old = node.node replace_object_state(new, old) node.node = new if isinstance(node.node, Var): # Handle them here just in case these aren't exposed through the AST. # TODO: Is this necessary? fixup_var(node.node, replacements)
def find_symbol_tables_recursive(prefix: str, symbols: SymbolTable) -> Dict[str, SymbolTable]: """Find all nested symbol tables. Args: prefix: Full name prefix (used for return value keys and to filter result so that cross references to other modules aren't included) symbols: Root symbol table Returns a dictionary from full name to corresponding symbol table. """ result = {} result[prefix] = symbols for name, node in symbols.items(): if isinstance(node.node, TypeInfo) and node.node.fullname().startswith(prefix + '.'): more = find_symbol_tables_recursive(prefix + '.' + name, node.node.names) result.update(more) return result
def replacement_map_from_symbol_table( old: SymbolTable, new: SymbolTable, prefix: str) -> Dict[SymbolNode, SymbolNode]: replacements = {} # type: Dict[SymbolNode, SymbolNode] for name, node in old.items(): if (name in new and (node.kind == MDEF or node.node and get_prefix(node.node.fullname()) == prefix)): new_node = new[name] if (type(new_node.node) == type(node.node) # noqa and new_node.node and node.node and new_node.node.fullname() == node.node.fullname() and new_node.kind == node.kind): replacements[new_node.node] = node.node if isinstance(node.node, TypeInfo) and isinstance(new_node.node, TypeInfo): type_repl = replacement_map_from_symbol_table( node.node.names, new_node.node.names, prefix) replacements.update(type_repl) return replacements
def snapshot_symbol_table(name_prefix: str, table: SymbolTable) -> Dict[str, SnapshotItem]: """Create a snapshot description that represents the state of a symbol table. The snapshot has a representation based on nested tuples and dicts that makes it easy and fast to find differences. Only "shallow" state is included in the snapshot -- references to things defined in other modules are represented just by the names of the targets. """ result = {} # type: Dict[str, SnapshotItem] for name, symbol in table.items(): node = symbol.node # TODO: cross_ref? fullname = node.fullname() if node else None common = (fullname, symbol.kind, symbol.module_public) if symbol.kind == MODULE_REF: # This is a cross-reference to another module. # If the reference is busted because the other module is missing, # the node will be a "stale_info" TypeInfo produced by fixup, # but that doesn't really matter to us here. result[name] = ('Moduleref', common) elif symbol.kind == TVAR: assert isinstance(node, TypeVarExpr) result[name] = ('TypeVar', node.variance, [snapshot_type(value) for value in node.values], snapshot_type(node.upper_bound)) elif isinstance(symbol.node, TypeAlias): result[name] = ('TypeAlias', symbol.node.alias_tvars, symbol.node.normalized, symbol.node.no_args, snapshot_optional_type(symbol.node.target)) else: assert symbol.kind != UNBOUND_IMPORTED if node and get_prefix(node.fullname()) != name_prefix: # This is a cross-reference to a node defined in another module. result[name] = ('CrossRef', common) else: result[name] = snapshot_definition(node, common) return result
def replace_nodes_in_symbol_table(symbols: SymbolTable, replacements: Dict[SymbolNode, SymbolNode]) -> None: for name, node in symbols.items(): if node.node: if node.node in replacements: new = replacements[node.node] new.__dict__ = node.node.__dict__ node.node = new # TODO: Other node types if isinstance(node.node, Var) and node.node.type: node.node.type.accept(TypeReplaceVisitor(replacements)) node.node.info = cast(TypeInfo, replacements.get(node.node.info, node.node.info)) else: # TODO: Other node types if isinstance(node.node, Var) and node.node.type: node.node.type.accept(TypeReplaceVisitor(replacements)) override = node.type_override if override: override.accept(TypeReplaceVisitor(replacements))
def visit_symbol_table(self, symtab: SymbolTable) -> None: # Copy the items because we may mutate symtab. for key, value in list(symtab.items()): cross_ref = value.cross_ref if cross_ref is not None: # Fix up cross-reference. del value.cross_ref if cross_ref in self.modules: value.node = self.modules[cross_ref] else: stnode = lookup_qualified_stnode(self.modules, cross_ref) assert stnode is not None, "Could not find cross-ref %s" % (cross_ref,) value.node = stnode.node value.type_override = stnode.type_override else: if isinstance(value.node, TypeInfo): # TypeInfo has no accept(). TODO: Add it? self.visit_type_info(value.node) elif value.node is not None: value.node.accept(self) if value.type_override is not None: value.type_override.accept(self.type_fixer)
def visit_symbol_table(self, symtab: SymbolTable) -> None: # Copy the items because we may mutate symtab. for key, value in list(symtab.items()): cross_ref = value.cross_ref if cross_ref is not None: # Fix up cross-reference. value.cross_ref = None if cross_ref in self.modules: value.node = self.modules[cross_ref] else: stnode = lookup_qualified_stnode(self.modules, cross_ref, self.allow_missing) if stnode is not None: value.node = stnode.node elif not self.allow_missing: assert stnode is not None, "Could not find cross-ref %s" % (cross_ref,) else: # We have a missing crossref in allow missing mode, need to put something value.node = missing_info(self.modules) else: if isinstance(value.node, TypeInfo): # TypeInfo has no accept(). TODO: Add it? self.visit_type_info(value.node) elif value.node is not None: value.node.accept(self)
class SemanticAnalyzer(NodeVisitor): """Semantically analyze parsed mypy files. The analyzer binds names and does various consistency checks for a parse tree. Note that type checking is performed as a separate pass. """ # Library search paths lib_path = None # Module name space modules = None # Global name space for current module globals = None # Names declared using "global" (separate set for each scope) global_decls = None # Module-local name space for current modules # TODO not needed? module_names = None # Class type variables (the scope is a single class definition) class_tvars = None # Local names locals = None # All classes, from name to info (TODO needed?) types = None stack = None # Function local/type variable stack TODO remove type = None # TypeInfo of enclosing class (or None) is_init_method = None # Are we now analysing __init__? is_function = None # Are we now analysing a function/method? block_depth = None # Depth of nested blocks loop_depth = None # Depth of breakable loops cur_mod_id = None # Current module id (or None) (phase 2) imports = None # Imported modules (during phase 2 analysis) errors = None # Keep track of generated errors def __init__(self, lib_path, errors): """Create semantic analyzer. Use lib_path to search for modules, and report compile errors using the Errors instance. """ self.stack = [None] self.locals = [] self.imports = set() self.type = None self.block_depth = 0 self.loop_depth = 0 self.types = TypeInfoMap() self.lib_path = lib_path self.errors = errors self.modules = {} self.class_tvars = None self.is_init_method = False self.is_function = False # # First pass of semantic analysis # def anal_defs(self, defs, fnam, mod_id): """Perform the first analysis pass. Resolve the full names of definitions and construct type info structures, but do not resolve inter-definition references such as base classes. """ self.cur_mod_id = mod_id self.errors.set_file(fnam) self.globals = SymbolTable() self.global_decls = [set()] # Add implicit definition of '__name__'. name_def = VarDef([Var("__name__", Any())], True) defs.insert(0, name_def) for d in defs: if isinstance(d, AssignmentStmt): self.anal_assignment_stmt(d) elif isinstance(d, FuncDef): self.anal_func_def(d) elif isinstance(d, OverloadedFuncDef): self.anal_overloaded_func_def(d) elif isinstance(d, TypeDef): self.anal_type_def(d) elif isinstance(d, VarDef): self.anal_var_def(d) elif isinstance(d, ForStmt): self.anal_for_stmt(d) # Add implicit definition of 'None' to builtins, as we cannot define a # variable with a None type explicitly. if mod_id == "builtins": none_def = VarDef([Var("None", NoneTyp())], True) defs.append(none_def) self.anal_var_def(none_def) def anal_assignment_stmt(self, s): for lval in s.lvalues: self.analyse_lvalue(lval, False, True) def anal_func_def(self, d): self.check_no_global(d.name(), d, True) d._full_name = self.qualified_name(d.name()) self.globals[d.name()] = SymbolTableNode(GDEF, d, self.cur_mod_id) def anal_overloaded_func_def(self, d): self.check_no_global(d.name(), d) d._full_name = self.qualified_name(d.name()) self.globals[d.name()] = SymbolTableNode(GDEF, d, self.cur_mod_id) def anal_type_def(self, d): self.check_no_global(d.name, d) d.full_name = self.qualified_name(d.name) info = TypeInfo({}, {}, d) info.set_line(d.line) self.types[d.full_name] = info d.info = info self.globals[d.name] = SymbolTableNode(GDEF, info, self.cur_mod_id) def anal_var_def(self, d): for v in d.items: self.check_no_global(v.name(), d) v._full_name = self.qualified_name(v.name()) self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) def anal_for_stmt(self, s): for n in s.index: self.analyse_lvalue(n, False, True) # # Second pass of semantic analysis # # Do the bulk of semantic analysis in this second and final semantic # analysis pass (other than type checking). def visit_file(self, file_node, fnam): self.errors.set_file(fnam) self.globals = file_node.names self.module_names = SymbolTable() self.cur_mod_id = file_node.full_name() if "builtins" in self.modules: self.globals["__builtins__"] = SymbolTableNode(MODULE_REF, self.modules["builtins"], self.cur_mod_id) defs = file_node.defs for d in defs: d.accept(self) def visit_func_def(self, defn): if self.type and not self.locals: defn.info = self.type if not defn.is_overload: if defn.name() in self.type.methods: self.name_already_defined(defn.name(), defn) self.type.methods[defn.name()] = defn if defn.name() == "__init__": self.is_init_method = True if defn.args == []: self.fail("Method must have at least one argument", defn) if self.locals: self.add_local_func(defn, defn) self.errors.push_function(defn.name()) self.analyse_function(defn) self.errors.pop_function() self.is_init_method = False def visit_overloaded_func_def(self, defn): t = [] for f in defn.items: f.is_overload = True f.accept(self) t.append(function_type(f)) defn.type = Overloaded(t) defn.type.line = defn.line if self.type: self.type.methods[defn.name()] = defn defn.info = self.type def analyse_function(self, defn): self.enter() self.add_func_type_variables_to_symbol_table(defn) if defn.type: defn.type = self.anal_type(defn.type) if isinstance(defn, FuncDef): fdef = defn if self.type: defn.type = (defn.type).with_name('"{}" of "{}"'.format(fdef.name(), self.type.name())) else: defn.type = (defn.type).with_name('"{}"'.format(fdef.name())) if self.type and (defn.type).arg_types != []: (defn.type).arg_types[0] = self_type(fdef.info) for init in defn.init: if init: init.rvalue.accept(self) for v in defn.args: self.add_local(v, defn) for init_ in defn.init: if init_: init_.lvalues[0].accept(self) # The first argument of a method is self. if self.type and defn.args: defn.args[0].is_self = True defn.body.accept(self) self.leave() def add_func_type_variables_to_symbol_table(self, defn): if defn.type: tt = defn.type names = self.type_var_names() items = (tt).variables.items for i in range(len(items)): name = items[i].name if name in names: self.name_already_defined(name, defn) self.add_type_var(self.locals[-1], name, -i - 1) names.add(name) def type_var_names(self): if not self.type: return set() else: return set(self.type.type_vars) def add_type_var(self, scope, name, id): scope[name] = SymbolTableNode(TVAR, None, None, None, id) def visit_type_def(self, defn): if self.locals or self.type: self.fail("Nested classes not supported yet", defn) return self.type = defn.info self.add_class_type_variables_to_symbol_table(self.type) has_base_class = False for i in range(len(defn.base_types)): defn.base_types[i] = self.anal_type(defn.base_types[i]) self.type.bases.append(defn.base_types[i]) has_base_class = has_base_class or self.is_instance_type(defn.base_types[i]) # Add 'object' as implicit base if there is no other base class. if not defn.is_interface and not has_base_class and defn.full_name != "builtins.object": obj = self.object_type() defn.base_types.insert(0, obj) self.type.bases.append(obj) if defn.base_types: bt = defn.base_types if isinstance(bt[0], Instance): defn.info.base = (bt[0]).type for t in bt[1:]: if isinstance(t, Instance): defn.info.add_interface((t).type) defn.defs.accept(self) self.class_tvars = None self.type = None def object_type(self): sym = self.lookup_qualified("__builtins__.object", None) return Instance(sym.node, []) def is_instance_type(self, t): return isinstance(t, Instance) and not (t).type.is_interface def add_class_type_variables_to_symbol_table(self, info): vars = info.type_vars if vars != []: self.class_tvars = SymbolTable() for i in range(len(vars)): self.add_type_var(self.class_tvars, vars[i], i + 1) def visit_import(self, i): if not self.check_import_at_toplevel(i): return for id, as_id in i.ids: if as_id != id: m = self.modules[id] self.globals[as_id] = SymbolTableNode(MODULE_REF, m, self.cur_mod_id) else: base = id.split(".")[0] m = self.modules[base] self.globals[base] = SymbolTableNode(MODULE_REF, m, self.cur_mod_id) def visit_import_from(self, i): if not self.check_import_at_toplevel(i): return m = self.modules[i.id] for id, as_id in i.names: node = m.names.get(id, None) if node: self.globals[as_id] = SymbolTableNode(node.kind, node.node, self.cur_mod_id) else: self.fail("Module has no attribute '{}'".format(id), i) def visit_import_all(self, i): if not self.check_import_at_toplevel(i): return m = self.modules[i.id] for name, node in m.names.items(): if not name.startswith("_"): self.globals[name] = SymbolTableNode(node.kind, node.node, self.cur_mod_id) def check_import_at_toplevel(self, c): if self.block_depth > 0: self.fail("Imports within blocks not supported yet", c) return False else: return True # # Statements # def visit_block(self, b): self.block_depth += 1 for s in b.body: s.accept(self) self.block_depth -= 1 def visit_block_maybe(self, b): if b: self.visit_block(b) def visit_var_def(self, defn): for i in range(len(defn.items)): defn.items[i].type = self.anal_type(defn.items[i].type) for v in defn.items: if self.locals: defn.kind = LDEF self.add_local(v, defn) elif self.type: v.info = self.type v.is_initialized_in_class = defn.init is not None self.type.vars[v.name()] = v elif v.name not in self.globals: defn.kind = GDEF self.add_var(v, defn) if defn.init: defn.init.accept(self) def anal_type(self, t): if t: a = TypeAnalyser(self.lookup_qualified, self.fail) return t.accept(a) else: return None def visit_assignment_stmt(self, s): for lval in s.lvalues: self.analyse_lvalue(lval) s.rvalue.accept(self) def analyse_lvalue(self, lval, nested=False, add_defs=False): if isinstance(lval, NameExpr): n = lval nested_global = not self.locals and self.block_depth > 0 and not self.type if (add_defs or nested_global) and n.name not in self.globals: # Define new global name. v = Var(n.name) v._full_name = self.qualified_name(n.name) v.is_ready = False # Type not inferred yet n.node = v n.is_def = True n.kind = GDEF n.full_name = v._full_name self.globals[n.name] = SymbolTableNode(GDEF, v, self.cur_mod_id) elif isinstance(n.node, Var) and n.is_def: v = n.node self.module_names[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) elif self.locals and n.name not in self.locals[-1] and n.name not in self.global_decls[-1]: # Define new local name. v = Var(n.name) n.node = v n.is_def = True n.kind = LDEF self.add_local(v, n) elif not self.locals and (self.type and n.name not in self.type.vars): # Define a new attribute. v = Var(n.name) v.info = self.type v.is_initialized_in_class = True n.node = v n.is_def = True self.type.vars[n.name] = v else: # Bind to an existing name. lval.accept(self) elif isinstance(lval, MemberExpr): if not add_defs: self.analyse_member_lvalue(lval) elif isinstance(lval, IndexExpr): if not add_defs: lval.accept(self) elif isinstance(lval, ParenExpr): self.analyse_lvalue((lval).expr, nested, add_defs) elif (isinstance(lval, TupleExpr) or isinstance(lval, ListExpr)) and not nested: items = (lval).items for i in items: self.analyse_lvalue(i, True, add_defs) else: self.fail("Invalid assignment target", lval) def analyse_member_lvalue(self, lval): lval.accept(self) if self.is_init_method and isinstance(lval.expr, NameExpr): node = (lval.expr).node if isinstance(node, Var) and (node).is_self and lval.name not in self.type.vars: lval.is_def = True v = Var(lval.name) v.info = self.type v.is_ready = False lval.def_var = v self.type.vars[lval.name] = v def visit_expression_stmt(self, s): s.expr.accept(self) def visit_return_stmt(self, s): if not self.locals: self.fail("'return' outside function", s) if s.expr: s.expr.accept(self) def visit_raise_stmt(self, s): if s.expr: s.expr.accept(self) def visit_yield_stmt(self, s): if not self.locals: self.fail("'yield' outside function", s) if s.expr: s.expr.accept(self) def visit_assert_stmt(self, s): if s.expr: s.expr.accept(self) def visit_operator_assignment_stmt(self, s): s.lvalue.accept(self) s.rvalue.accept(self) def visit_while_stmt(self, s): s.expr.accept(self) self.loop_depth += 1 s.body.accept(self) self.loop_depth -= 1 self.visit_block_maybe(s.else_body) def visit_for_stmt(self, s): s.expr.accept(self) # Bind index variables and check if they define new names. for n in s.index: self.analyse_lvalue(n) # Analyze index variable types. for i in range(len(s.types)): t = s.types[i] if t: s.types[i] = self.anal_type(t) v = s.index[i].node # TODO check if redefinition v.type = s.types[i] # Report error if only some of the loop variables have annotations. if s.types != [None] * len(s.types) and None in s.types: self.fail("Cannot mix unannotated and annotated loop variables", s) self.loop_depth += 1 self.visit_block(s.body) self.loop_depth -= 1 self.visit_block_maybe(s.else_body) def visit_break_stmt(self, s): if self.loop_depth == 0: self.fail("'break' outside loop", s) def visit_continue_stmt(self, s): if self.loop_depth == 0: self.fail("'continue' outside loop", s) def visit_if_stmt(self, s): for i in range(len(s.expr)): s.expr[i].accept(self) self.visit_block(s.body[i]) self.visit_block_maybe(s.else_body) def visit_try_stmt(self, s): s.body.accept(self) for i in range(len(s.types)): if s.types[i]: s.types[i].accept(self) if s.vars[i]: self.add_var(s.vars[i], s.vars[i]) s.handlers[i].accept(self) self.visit_block_maybe(s.else_body) self.visit_block_maybe(s.finally_body) def visit_with_stmt(self, s): for e in s.expr: e.accept(self) for n in s.name: if n: self.add_var(n, s) self.visit_block(s.body) def visit_del_stmt(self, s): s.expr.accept(self) if not isinstance(s.expr, IndexExpr): self.fail("Invalid delete target", s) def visit_global_decl(self, g): for n in g.names: self.global_decls[-1].add(n) # # Expressions # def visit_name_expr(self, expr): n = self.lookup(expr.name, expr) if n: if n.kind == TVAR: self.fail("'{}' is a type variable and only valid in type " "context".format(expr.name), expr) else: expr.kind = n.kind expr.node = n.node expr.full_name = n.full_name() def visit_super_expr(self, expr): if not self.type: self.fail('"super" used outside class', expr) return expr.info = self.type def visit_tuple_expr(self, expr): for item in expr.items: item.accept(self) if expr.types: for i in range(len(expr.types)): expr.types[i] = self.anal_type(expr.types[i]) def visit_list_expr(self, expr): for item in expr.items: item.accept(self) expr.type = self.anal_type(expr.type) def visit_dict_expr(self, expr): for key, value in expr.items: key.accept(self) value.accept(self) expr.key_type = self.anal_type(expr.key_type) expr.value_type = self.anal_type(expr.value_type) def visit_paren_expr(self, expr): expr.expr.accept(self) def visit_call_expr(self, expr): expr.callee.accept(self) for a in expr.args: a.accept(self) def visit_member_expr(self, expr): base = expr.expr base.accept(self) # Bind references to module attributes. if isinstance(base, RefExpr) and (base).kind == MODULE_REF: names = ((base).node).names n = names.get(expr.name, None) if n: expr.kind = n.kind expr.full_name = n.full_name() expr.node = n.node else: self.fail("Module has no attribute '{}'".format(expr.name), expr) def visit_op_expr(self, expr): expr.left.accept(self) expr.right.accept(self) def visit_unary_expr(self, expr): expr.expr.accept(self) def visit_index_expr(self, expr): expr.base.accept(self) expr.index.accept(self) def visit_slice_expr(self, expr): if expr.begin_index: expr.begin_index.accept(self) if expr.end_index: expr.end_index.accept(self) if expr.stride: expr.stride.accept(self) def visit_cast_expr(self, expr): expr.expr.accept(self) expr.type = self.anal_type(expr.type) def visit_type_application(self, expr): expr.expr.accept(self) for i in range(len(expr.types)): expr.types[i] = self.anal_type(expr.types[i]) def visit_list_comprehension(self, expr): expr.generator.accept(self) def visit_generator_expr(self, expr): self.enter() expr.right_expr.accept(self) # Bind index variables. for n in expr.index: self.analyse_lvalue(n) if expr.condition: expr.condition.accept(self) # TODO analyze variable types (see visit_for_stmt) expr.left_expr.accept(self) self.leave() def visit_func_expr(self, expr): self.analyse_function(expr) # # Helpers # def lookup(self, name, ctx): if name in self.global_decls[-1]: # Name declared using 'global x' takes precedence. if name in self.globals: return self.globals[name] else: self.name_not_defined(name, ctx) return None if self.locals: for table in reversed(self.locals): if name in table: return table[name] if self.class_tvars and name in self.class_tvars: return self.class_tvars[name] if self.type and (not self.locals and self.type.has_readable_member(name)): # Reference to attribute within class body. v = self.type.get_var(name) if v: return SymbolTableNode(MDEF, v, typ=v.type) m = self.type.get_method(name) return SymbolTableNode(MDEF, m, typ=m.type) if name in self.globals: return self.globals[name] else: b = self.globals.get("__builtins__", None) if b: table = (b.node).names if name in table: return table[name] if self.type and (not self.locals and self.type.has_readable_member(name)): self.fail("Feature not implemented yet (class attributes)", ctx) return None self.name_not_defined(name, ctx) return None def lookup_qualified(self, name, ctx): if "." not in name: return self.lookup(name, ctx) else: parts = name.split(".") n = self.lookup(parts[0], ctx) if n: for i in range(1, len(parts)): if isinstance(n.node, TypeInfo): self.fail("Feature not implemented yet (class attributes)", ctx) return None n = (n.node).names.get(parts[i], None) if not n: self.name_not_defined(name, ctx) return n def qualified_name(self, n): return self.cur_mod_id + "." + n def enter(self): self.locals.append(SymbolTable()) self.global_decls.append(set()) def leave(self): self.locals.pop() self.global_decls.pop() def add_var(self, v, ctx): if self.locals: self.add_local(v, ctx) else: self.globals[v.name()] = SymbolTableNode(GDEF, v, self.cur_mod_id) v._full_name = self.qualified_name(v.name()) def add_local(self, v, ctx): if v.name() in self.locals[-1]: self.name_already_defined(v.name(), ctx) v._full_name = v.name() self.locals[-1][v.name()] = SymbolTableNode(LDEF, v) def add_local_func(self, defn, ctx): # TODO combine with above if not defn.is_overload and defn.name() in self.locals[-1]: self.name_already_defined(defn.name(), ctx) defn._full_name = defn.name() self.locals[-1][defn.name()] = SymbolTableNode(LDEF, defn) def check_no_global(self, n, ctx, is_func=False): if n in self.globals: if is_func and isinstance(self.globals[n].node, FuncDef): self.fail( ("Name '{}' already defined (overload variants " "must be next to each other)").format(n), ctx ) else: self.name_already_defined(n, ctx) def name_not_defined(self, name, ctx): self.fail("Name '{}' is not defined".format(name), ctx) def name_already_defined(self, name, ctx): self.fail("Name '{}' already defined".format(name), ctx) def fail(self, msg, ctx): self.errors.report(ctx.get_line(), msg)
def analyze_symbol_table(self, names: SymbolTable) -> None: """Analyze types in symbol table nodes only (shallow).""" for node in names.values(): if node.type_override: self.analyze(node.type_override, node)
def analyze_symbol_table(self, names: SymbolTable) -> None: """Analyze types in symbol table nodes only (shallow).""" for node in names.values(): if isinstance(node.node, TypeAlias): self.analyze(node.node.target, node.node)