def test_getitem__concrete_index(self): t = self._ctx.convert.tuple_to_value((self._var, )) index = self._convert.constant_to_var(0) node, var = t.cls.getitem_slot(self._node, index) self.assertIs(node, self._node) self.assertIs(abstract_utils.get_atomic_value(var), abstract_utils.get_atomic_value(self._var))
def make_class(self, node, bases, f_locals): # If BuildClass.call() hits max depth, f_locals will be [unsolvable] # Since we don't support defining NamedTuple subclasses in a nested scope # anyway, we can just return unsolvable here to prevent a crash, and let the # invalid namedtuple error get raised later. if isinstance(f_locals.data[0], abstract.Unsolvable): return node, self.ctx.new_unsolvable(node) f_locals = abstract_utils.get_atomic_python_constant(f_locals) # retrieve __qualname__ to get the name of class name = f_locals["__qualname__"] nameval = abstract_utils.get_atomic_python_constant(name) if "." in nameval: nameval = nameval.rsplit(".", 1)[-1] name = self.ctx.convert.constant_to_var(nameval) # assemble the arguments that are compatible with NamedTupleFuncBuilder.call field_list = [] defaults = [] cls_locals = classgen.get_class_locals( nameval, allow_methods=True, ordering=classgen.Ordering.FIRST_ANNOTATE, ctx=self.ctx) for k, local in cls_locals.items(): assert local.typ if k in f_locals: defaults.append(f_locals[k]) k = self.ctx.convert.constant_to_var(k, node=node) field_list.append(self.ctx.convert.build_tuple(node, (k, local.typ))) anno = self.ctx.convert.build_list(node, field_list) posargs = (name, anno) args = function.Args(posargs=posargs) node, cls_var = self.namedtuple.call(node, None, args, bases) cls_val = abstract_utils.get_atomic_value(cls_var) if not isinstance(cls_val, abstract.Unsolvable): # set __new__.__defaults__ defaults = self.ctx.convert.build_tuple(node, defaults) node, new_attr = self.ctx.attribute_handler.get_attribute( node, cls_val, "__new__") new_attr = abstract_utils.get_atomic_value(new_attr) node = self.ctx.attribute_handler.set_attribute(node, new_attr, "__defaults__", defaults) # set the attribute without overriding special namedtuple attributes node, fields = self.ctx.attribute_handler.get_attribute( node, cls_val, "_fields") fields = abstract_utils.get_atomic_python_constant(fields, tuple) fields = [abstract_utils.get_atomic_python_constant(field, str) for field in fields] for key in f_locals: if key in self._prohibited: self.ctx.errorlog.not_writable(self.ctx.vm.frames, cls_val, key) if key not in abstract_utils.CLASS_LEVEL_IGNORE and key not in fields: node = self.ctx.attribute_handler.set_attribute( node, cls_val, key, f_locals[key]) return node, cls_var
def test_instantiate_interpreter_class(self): cls = abstract.InterpreterClass("X", [], {}, None, self._ctx) # When there is no current frame, create a new instance every time. v1 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) v2 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) self.assertIsNot(v1, v2) # Create one instance per opcode. fake_opcode = object() self._ctx.vm.push_frame(frame_state.SimpleFrame(fake_opcode)) v3 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) v4 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) self.assertIsNot(v1, v3) self.assertIsNot(v2, v3) self.assertIs(v3, v4)
def generator(): """Yields mutations.""" if (not (self.is_attribute_of_class or self.name == "__new__") or not first_arg or not substs): return try: inst = abstract_utils.get_atomic_value(first_arg, _instance_base.Instance) except abstract_utils.ConversionError: return if inst.cls.template: for subst in substs: for k, v in subst.items(): if k in inst.instance_type_parameters: value = inst.instance_type_parameters[ k].AssignToNewVariable(node) if all( isinstance(val, _singletons.Unknown) for val in v.data): for param in inst.cls.template: if subst.same_name(k, param.full_name): value.PasteVariable( param.instantiate(node), node) break else: # See GenericFeatureTest.test_reinherit_generic in # tests/test_generic2. This can happen if one generic class # inherits from another and separately reuses a TypeVar. value.PasteVariable(v, node) else: value.PasteVariable(v, node) yield function.Mutation(inst, k, value)
def getitem_slot(self, node, index_var): """Implementation of tuple.__getitem__.""" try: index = self.ctx.convert.value_to_constant( abstract_utils.get_atomic_value(index_var), (int, slice)) except abstract_utils.ConversionError: pass else: if isinstance(index, slice): if self._instance: slice_content = self._instance.pyval[index] return node, self.ctx.convert.build_tuple( node, slice_content) else: # Constructing the tuple directly is faster than calling call_pytd. instance = _instance_base.Instance( self.ctx.convert.tuple_type, self.ctx) node, contained_type = self.ctx.vm.init_class( node, self.formal_type_parameters[abstract_utils.T]) instance.merge_instance_type_parameter( node, abstract_utils.T, contained_type) return node, instance.to_variable(node) if -self.tuple_length <= index < self.tuple_length: # Index out of bounds is not a pytype error because of the high # likelihood of false positives, e.g., # tup = [] # idx = 0 # if idx < len(tup): # tup[idx] return node, self._instantiate_index(node, index) return self.call_pytd(node, "__getitem__", self.instantiate(node), index_var)
def get_decorated_method(name, value, func_slot): fvar = getattr(value, func_slot) func = abstract_utils.get_atomic_value(fvar, abstract.Function) defn = self.value_to_pytd_def(node, func, name) defn = defn.Visit(visitors.DropMutableParameters()) defn = add_final(defn, value) return defn
def __init__(self, callself, underlying): super().__init__(underlying.name, underlying.ctx) self.cls = underlying.cls self._callself = callself self.underlying = underlying self.is_attribute_of_class = False self.is_class_builder = False # If the function belongs to `ParameterizedClass`, we will annotate the # `self` when do argument matching self.replace_self_annot = None inst = abstract_utils.get_atomic_value( self._callself, default=self.ctx.convert.unsolvable) if self._should_replace_self_annot(): if (isinstance(inst.cls, class_mixin.Class) and inst.cls.full_name != "builtins.type"): for cls in inst.cls.mro: if isinstance(cls, _classes.ParameterizedClass): base_cls = cls.base_cls else: base_cls = cls if isinstance(base_cls, class_mixin.Class) and base_cls.template: self.replace_self_annot = ( _classes.ParameterizedClass. get_generic_instance_type(base_cls)) break if isinstance(inst, _instance_base.SimpleValue): self.alias_map = inst.instance_type_parameters.uf elif isinstance(inst, _typing.TypeParameterInstance): self.alias_map = inst.instance.instance_type_parameters.uf else: self.alias_map = None
def call(self, node, unused_func, args): """Adds a metaclass.""" self.match_args(node, args) meta = abstract_utils.get_atomic_value( args.posargs[0], default=self.ctx.convert.unsolvable) return node, AddMetaclassInstance(meta, self.ctx, self.module_name).to_variable(node)
def _inner_cls_check(self, last_frame): """Check if the function and its nested class use same type parameter.""" # get all type parameters from function annotations all_type_parameters = [] for annot in self.signature.annotations.values(): params = self.ctx.annotation_utils.get_type_parameters(annot) all_type_parameters.extend(itm.with_module(None) for itm in params) if all_type_parameters: for key, value in last_frame.f_locals.pyval.items(): value = abstract_utils.get_atomic_value( value, default=self.ctx.convert.unsolvable) if (isinstance(value, _classes.InterpreterClass) and value.template and key == value.name): # `value` is a nested class definition. inner_cls_types = value.collect_inner_cls_types() inner_cls_types.update([(value, item.with_module(None)) for item in value.template]) # Report errors in a deterministic order. for cls, item in sorted(inner_cls_types, key=lambda typ: typ[1].name): if item in all_type_parameters: self.ctx.errorlog.invalid_annotation( self.ctx.vm.simple_stack( self.get_first_opcode()), item, ("Function [%s] and its nested generic class [%s] cannot use " "the same type variable %s") % (self.full_name, cls.full_name, item.name))
def call(self, node, func, args): args = args.simplify(node, self.ctx) self.match_args(node, args, match_all_views=True) # As long as the types match we do not really care about the actual # class name. But, if we have a string literal value as the name arg, # we will use it. name_arg = args.namedargs.get(self._name_arg_name) or args.posargs[0] try: _ = abstract_utils.get_atomic_python_constant(name_arg, str) except abstract_utils.ConversionError: name_arg = self.ctx.convert.constant_to_var( f"_NewType_Internal_Class_Name_{self.internal_name_counter}_") type_arg = args.namedargs.get(self._type_arg_name) or args.posargs[1] try: type_value = abstract_utils.get_atomic_value(type_arg) except abstract_utils.ConversionError: # We need the type arg to be an atomic value. If not, we just # silently return unsolvable. return node, self.ctx.new_unsolvable(node) value_arg_name = "val" constructor = overlay_utils.make_method( self.ctx, node, name="__init__", params=[Param(value_arg_name, type_value)]) members = abstract.Dict(self.ctx) members.set_str_item(node, "__init__", constructor) return self.ctx.make_class(node, name_arg, (type_arg, ), members.to_variable(node), None)
def test_call_wrong_argcount(self): self._ctx.vm.push_frame(frame_state.SimpleFrame()) node, result = self._is_instance.call( self._node, None, function.Args((), self.new_dict(), None, None)) self.assertEqual(self._node, node) self.assertIsInstance(abstract_utils.get_atomic_value(result), abstract.Unsolvable) self.assertRegex(str(self._ctx.errorlog), "missing-parameter")
def call(self, node, unused_func, args): """Creates an anonymous class to act as a metaclass.""" self.match_args(node, args) meta = abstract_utils.get_atomic_value( args.posargs[0], default=self.ctx.convert.unsolvable) bases = args.posargs[1:] result = WithMetaclassInstance(self.ctx, meta, bases).to_variable(node) return node, result
def _eval_expr_as_tuple(self, node, expr, stack): """Evaluate an expression as a tuple.""" if not expr: return (), None f_globals = self.ctx.vm.frame.f_globals f_locals = self.ctx.vm.frame.f_locals with self.ctx.vm.generate_late_annotations(stack): result_var, errorlog = abstract_utils.eval_expr(self.ctx, node, f_globals, f_locals, expr) result = abstract_utils.get_atomic_value(result_var) # If the result is a tuple, expand it. if (isinstance(result, mixin.PythonConstant) and isinstance(result.pyval, tuple)): return (tuple(abstract_utils.get_atomic_value(x) for x in result.pyval), errorlog) else: return (result,), errorlog
def get_inner_classes(self): """Return the list of top-level nested classes.""" values = [ abstract_utils.get_atomic_value( mbr, default=self.ctx.convert.unsolvable) for mbr in self.members.values() ] return [ x for x in values if isinstance(x, InterpreterClass) and x != self ]
def test_call_wrong_keywords(self): self._ctx.vm.push_frame(frame_state.SimpleFrame()) x = self.new_var(abstract.Unknown(self._ctx)) node, result = self._is_instance.call( self._node, None, function.Args( (x, x), self.new_dict(foo=x), None, None)) self.assertEqual(self._node, node) self.assertIsInstance(abstract_utils.get_atomic_value(result), abstract.Unsolvable) self.assertRegex( str(self._ctx.errorlog), r"foo.*isinstance.*\[wrong-keyword-args\]")
def to_annotation_container(self): if _isinstance(self, "PyTDClass") and self.full_name == "builtins.tuple": # If we are parameterizing builtins.tuple, replace it with typing.Tuple so # that heterogeneous tuple annotations work. We need the isinstance() # check to distinguish PyTDClass(tuple) from ParameterizedClass(tuple); # the latter appears here when a generic type alias is being substituted. typing = self.ctx.vm.import_module("typing", "typing", 0).get_module("Tuple") typing.load_lazy_attribute("Tuple") return abstract_utils.get_atomic_value(typing.members["Tuple"]) return _make("AnnotationContainer", self.name, self.ctx, self)
def collect_inner_cls_types(self, max_depth=5): """Collect all the type parameters from nested classes.""" templates = set() if max_depth > 0: for mbr in self.members.values(): mbr = abstract_utils.get_atomic_value( mbr, default=self.ctx.convert.unsolvable) if isinstance(mbr, InterpreterClass) and mbr.template: templates.update([(mbr, item.with_module(None)) for item in mbr.template]) templates.update(mbr.collect_inner_cls_types(max_depth - 1)) return templates
def _check_str_key_value(self, node, name, value_var): if not self._check_str_key(name): return typ = abstract_utils.get_atomic_value(self.fields[name]) bad = self.ctx.matcher(node).bad_matches(value_var, typ) for view, error_details in bad: binding = view[value_var] self.ctx.errorlog.annotation_type_mismatch(self.ctx.vm.frames, typ, binding, name, error_details, typed_dict=self)
def set_class(self, node, var): """Set the __class__ of an instance, for code that does "x.__class__ = y.""" # Simplification: Setting __class__ is done rarely, and supporting this # action would complicate pytype considerably by forcing us to track the # class in a variable, so we instead fall back to Any. try: new_cls = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.cls = self.ctx.convert.unsolvable else: if self.cls != new_cls: self.cls = self.ctx.convert.unsolvable return node
def _get_scopes( state, names: Sequence[str], ctx, ) -> Sequence[Union[abstract.InterpreterClass, abstract.InterpreterFunction]]: """Gets the class or function objects for a sequence of nested scope names. For example, if the code under analysis is: class Foo: def f(self): def g(): ... then when called with ['Foo', 'f', 'g'], this method returns [InterpreterClass(Foo), InterpreterFunction(f), InterpreterFunction(g)]. Arguments: state: The current state. names: A sequence of names for consecutive nested scopes in the module under analysis. Must start with a module-level name. ctx: The current context. Returns: The class or function object corresponding to each name in 'names'. """ scopes = [] for name in names: prev = scopes[-1] if scopes else None if not prev: try: _, var = ctx.vm.load_global(state, name) except KeyError: break elif isinstance(prev, abstract.InterpreterClass): if name in prev.members: var = prev.members[name] else: break else: assert isinstance(prev, abstract.InterpreterFunction) # For last_frame to be populated, 'prev' has to have been called at # least once. This has to be true for all functions except the innermost # one, since pytype cannot detect a nested function without analyzing # the code that defines the nested function. if prev.last_frame and name in prev.last_frame.f_locals.pyval: var = prev.last_frame.f_locals.pyval[name] else: break try: scopes.append(abstract_utils.get_atomic_value( var, (abstract.InterpreterClass, abstract.InterpreterFunction))) except abstract_utils.ConversionError: break return scopes
def _update_signature_scope(self): # If this is a nested function in an instance method and the nested function # accesses 'self', then the first variable in the closure is 'self'. We use # 'self' to update the scopes of any type parameters in the nested method's # signature to the containing class. if not self.closure: return maybe_instance = self.closure[0] try: instance = abstract_utils.get_atomic_value(maybe_instance, _instance_base.Instance) except abstract_utils.ConversionError: return if isinstance(instance.cls, _classes.InterpreterClass): instance.cls.update_signature_scope(self)
def _load_all_formal_type_parameters(self): """Load _all_formal_type_parameters.""" if self._all_formal_type_parameters_loaded: return bases = [ abstract_utils.get_atomic_value( base, default=self.ctx.convert.unsolvable) for base in self.bases() ] for base in bases: abstract_utils.parse_formal_type_parameters( base, self.full_name, self._all_formal_type_parameters) self._all_formal_type_parameters_loaded = True
def test_callable_no_args(self): ast = self._load_ast("a", """ from typing import Callable x = ... # type: Callable[[], ...] """) x = ast.Lookup("a.x").type cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node) instance = self._ctx.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._ctx.root_node) self.assertIsInstance( cls.get_formal_type_parameter(abstract_utils.ARGS), abstract.Empty) self.assertEqual( abstract_utils.get_atomic_value( instance.get_instance_type_parameter(abstract_utils.ARGS)), self._ctx.convert.empty)
def property_get(self, callself, is_class=False): if self.kind == pytd.MethodKind.STATICMETHOD: if is_class: # Binding the function to None rather than not binding it tells # output.py to infer the type as a Callable rather than reproducing the # signature, including the @staticmethod decorator, which is # undesirable for module-level aliases. callself = None return _function_base.StaticMethod(self.name, self, callself, self.ctx) elif self.kind == pytd.MethodKind.CLASSMETHOD: if not is_class: callself = abstract_utils.get_atomic_value( callself, default=self.ctx.convert.unsolvable) if isinstance(callself, _typing.TypeParameterInstance): callself = abstract_utils.get_atomic_value( callself.instance.get_instance_type_parameter(callself.name), default=self.ctx.convert.unsolvable) # callself is the instance, and we want to bind to its class. callself = callself.cls.to_variable(self.ctx.root_node) return _function_base.ClassMethod(self.name, self, callself, self.ctx) elif self.kind == pytd.MethodKind.PROPERTY and not is_class: return _function_base.Property(self.name, self, callself, self.ctx) else: return super().property_get(callself, is_class)
def _get_template(val: Any): """Get the value's class template.""" if _isinstance(val, "Class"): res = {t.full_name for t in val.template} if _isinstance(val, "ParameterizedClass"): res.update(_get_template(val.base_cls)) elif _isinstance(val, ("PyTDClass", "InterpreterClass")): for base in val.bases(): base = abstract_utils.get_atomic_value( base, default=val.ctx.convert.unsolvable) res.update(_get_template(base)) return res elif val.cls != val: return _get_template(val.cls) else: return set()
def _getargs(self, node, args, functional): self.match_args(node, args) sig, = self.signatures callargs = {name: var for name, var, _ in sig.signature.iter_args(args)} # typing.NamedTuple doesn't support rename or verbose name_var = callargs["typename"] fields_var = callargs["fields"] fields = abstract_utils.get_atomic_python_constant(fields_var) if isinstance(fields, str): # Since str matches Sequence, we have to manually check for it. raise function.WrongArgTypes(sig.signature, args, self.ctx, self._fields_param) # The fields is a list of tuples, so we need to deeply unwrap them. fields = [abstract_utils.get_atomic_python_constant(t) for t in fields] # We need the actual string for the field names and the BaseValue # for the field types. names = [] types = [] for field in fields: if isinstance(field, str): # Since str matches Sequence, we have to manually check for it. raise function.WrongArgTypes(sig.signature, args, self.ctx, self._fields_param) if (len(field) != 2 or any(not self._is_str_instance(v) for v in field[0].data)): # Note that we don't need to check field[1] because both 'str' # (forward reference) and 'type' are valid for it. raise function.WrongArgTypes(sig.signature, args, self.ctx, self._fields_param) name, typ = field name_py_constant = abstract_utils.get_atomic_python_constant(name) names.append(name_py_constant) if functional: allowed_type_params = ( self.ctx.annotation_utils.get_callable_type_parameter_names(typ)) annot = self.ctx.annotation_utils.extract_annotation( node, typ, name_py_constant, self.ctx.vm.simple_stack(), allowed_type_params=allowed_type_params) else: # This NamedTuple was constructed with the class syntax. The field # annotations were already processed when added to __annotations__. annot = abstract_utils.get_atomic_value(typ) types.append(annot) return name_var, names, types
def _make_init(self, props): # __init__ method for type checking signatures. # We construct this here and pass it to TypedDictClass because we need # access to abstract.SignedFunction. sig = function.Signature.from_param_names( f"{props.name}.__init__", props.fields.keys(), kind=pytd.ParameterKind.KWONLY) sig.annotations = { k: abstract_utils.get_atomic_value(v) for k, v in props.fields.items() } sig.defaults = { k: self.ctx.new_unsolvable(self.ctx.root_node) for k in props.optional } return abstract.SignedFunction(sig, self.ctx)
def update_method_type_params(self): if self.template: # For function type parameters check for mbr in self.members.values(): m = abstract_utils.get_atomic_value( mbr, default=self.ctx.convert.unsolvable) if _isinstance(m, "SignedFunction"): self.update_signature_scope(m) elif mbr.data and all( x.__class__.__name__ == "PropertyInstance" for x in mbr.data): # We generate a new variable every time we add a property slot, so we # take the last one (which contains bindings for all defined slots). prop = mbr.data[-1] for slot in (prop.fget, prop.fset, prop.fdel): if slot: for d in slot.data: if _isinstance(d, "SignedFunction"): self.update_signature_scope(d)
def extract_annotation( self, node, var, name, stack, allowed_type_params=None): """Returns an annotation extracted from 'var'. Args: node: The current node. var: The variable to extract from. name: The annotated name. stack: The frame stack. allowed_type_params: Type parameters that are allowed to appear in the annotation. 'None' means all are allowed. """ try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.ctx.errorlog.ambiguous_annotation(self.ctx.vm.frames, None, name) return self.ctx.convert.unsolvable typ = self._process_one_annotation(node, typ, name, stack) if not typ: return self.ctx.convert.unsolvable if typ.formal and allowed_type_params is not None: illegal_params = [x.name for x in self.get_type_parameters(typ) if x.name not in allowed_type_params] if illegal_params: details = "TypeVar(s) %s not in scope" % ", ".join( repr(p) for p in utils.unique_list(illegal_params)) if self.ctx.vm.frame.func: method = self.ctx.vm.frame.func.data if isinstance(method, abstract.BoundFunction): desc = "class" frame_name = method.name.rsplit(".", 1)[0] else: desc = "class" if method.is_class_builder else "method" frame_name = method.name details += f" for {desc} {frame_name!r}" if "AnyStr" in illegal_params: str_type = "Union[str, bytes]" details += (f"\nNote: For all string types, use {str_type}.") self.ctx.errorlog.invalid_annotation(stack, typ, details, name) return self.ctx.convert.unsolvable return typ
def _constant_to_value(self, pyval, subst, get_node): """Create a BaseValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if isinstance(pyval, str): return abstract.ConcreteValue(pyval, self.str_type, self.ctx) elif isinstance(pyval, bytes): return abstract.ConcreteValue(pyval, self.bytes_type, self.ctx) elif isinstance(pyval, bool): return self.true if pyval else self.false elif isinstance(pyval, int) and -1 <= pyval <= _MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.ConcreteValue(pyval, self.int_type, self.ctx) elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif pyval.__class__ is frozenset: instance = abstract.Instance(self.frozenset_type, self.ctx) for element in pyval: instance.merge_instance_type_parameter( self.ctx.root_node, abstract_utils.T, self.constant_to_var(element, subst, self.ctx.root_node)) return instance elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.ConcreteValue(pyval, self.primitive_classes[types.CodeType], self.ctx) elif pyval is super: return special_builtins.Super(self.ctx) elif pyval is object: return special_builtins.Object(self.ctx) elif pyval.__class__ is type: try: return self.name_to_value(self._type_to_name(pyval), subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.LateType): actual = self._load_late_type(pyval) return self._constant_to_value(actual, subst, get_node) elif isinstance(pyval, pytd.TypeDeclUnit): return self._create_module(pyval) elif isinstance(pyval, pytd.Module): mod = self.ctx.loader.import_name(pyval.module_name) return self._create_module(mod) elif isinstance(pyval, pytd.Class): if pyval.name == "builtins.super": return self.ctx.special_builtins["super"] elif pyval.name == "builtins.object": return self.object_type elif pyval.name == "types.ModuleType": return self.module_type elif pyval.name == "_importlib_modulespec.ModuleType": # Python 3's typeshed uses a stub file indirection to define ModuleType # even though it is exported via types.pyi. return self.module_type elif pyval.name == "types.FunctionType": return self.function_type else: module, dot, base_name = pyval.name.rpartition(".") # typing.TypingContainer intentionally loads the underlying pytd types. if (module not in ("typing", "typing_extensions") and module in overlay_dict.overlays): overlay = self.ctx.vm.import_module(module, module, 0) if overlay.get_module(base_name) is overlay: overlay.load_lazy_attribute(base_name) return abstract_utils.get_atomic_value(overlay.members[base_name]) try: cls = abstract.PyTDClass.make(base_name, pyval, self.ctx) except mro.MROError as e: self.ctx.errorlog.mro_error(self.ctx.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module cls.call_metaclass_init(get_node()) return cls elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.ctx) for sig in pyval.signatures ] type_new = self.ctx.loader.lookup_builtin("builtins.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.ctx) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): if pyval.cls: cls = pyval.cls else: # If pyval is a reference to a class in builtins or typing, we can fill # in the class ourselves. lookup_builtin raises a KeyError if the name # is not found. cls = self.ctx.loader.lookup_builtin(pyval.name) assert isinstance(cls, pytd.Class) return self.constant_to_value(cls, subst, self.ctx.root_node) elif isinstance(pyval, pytd.NothingType): return self.empty elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.AnythingType)): # We allow "X = ... # type: Any" to declare X as a type. return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.GenericType) and pyval.type.name == "builtins.type"): # `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`. param, = pyval.type.parameters return self.constant_to_value(param, subst, self.ctx.root_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.ctx.root_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.ctx) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.ctx.root_node) for c in pyval.constraints) bound = ( pyval.bound and self.constant_to_value(pyval.bound, {}, self.ctx.root_node)) return abstract.TypeParameter( pyval.name, self.ctx, constraints=constraints, bound=bound, module=pyval.scope) elif isinstance(pyval, abstract_utils.AsInstance): cls = pyval.cls if isinstance(cls, pytd.LateType): actual = self._load_late_type(cls) if not isinstance(actual, pytd.ClassType): return self.unsolvable cls = actual.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar": param, = cls.parameters return self.constant_to_value( abstract_utils.AsInstance(param), subst, self.ctx.root_node) elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and cls.template): # If we're converting a generic Class, need to create a new instance of # it. See test_classes.testGenericReinstantiated. if isinstance(cls, pytd.Class): params = tuple(t.type_param.upper_value for t in cls.template) cls = pytd.GenericType(base_type=pytd.ClassType(cls.name, cls), parameters=params) if isinstance(cls.base_type, pytd.LateType): actual = self._load_late_type(cls.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base_cls = actual.cls else: base_type = cls.base_type assert isinstance(base_type, pytd.ClassType) base_cls = base_type.cls assert isinstance(base_cls, pytd.Class), base_cls if base_cls.name == "builtins.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.full_name not in subst: raise self.TypeParameterError(c.full_name) # deformalize gets rid of any unexpected TypeVars, which can appear # if something is annotated as Type[T]. return self.ctx.annotation_utils.deformalize( self.merge_classes(subst[c.full_name].data)) else: return self.constant_to_value(c, subst, self.ctx.root_node) elif isinstance(cls, pytd.TupleType): content = tuple(self.constant_to_var(abstract_utils.AsInstance(p), subst, get_node()) for p in cls.parameters) return self.tuple_to_value(content) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.ctx.root_node) return abstract.Instance(clsval, self.ctx) else: clsval = self.constant_to_value(base_cls, subst, self.ctx.root_node) instance = abstract.Instance(clsval, self.ctx) num_params = len(cls.parameters) assert num_params <= len(base_cls.template) for i, formal in enumerate(base_cls.template): if i < num_params: node = get_node() p = self.constant_to_var( abstract_utils.AsInstance(cls.parameters[i]), subst, node) else: # An omitted type parameter implies `Any`. node = self.ctx.root_node p = self.unsolvable.to_variable(node) instance.merge_instance_type_parameter(node, formal.name, p) return instance elif isinstance(cls, pytd.Class): assert not cls.template # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in ["builtins.type", "builtins.property"]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.ctx.root_node) instance = abstract.Instance(mycls, self.ctx) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] elif isinstance(cls, pytd.Literal): return self.constant_to_value( self._get_literal_value(cls.value), subst, self.ctx.root_node) else: return self.constant_to_value(cls, subst, self.ctx.root_node) elif (isinstance(pyval, pytd.GenericType) and pyval.name == "typing.ClassVar"): param, = pyval.parameters return self.constant_to_value(param, subst, self.ctx.root_node) elif isinstance(pyval, pytd.GenericType): if isinstance(pyval.base_type, pytd.LateType): actual = self._load_late_type(pyval.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base = actual.cls else: assert isinstance(pyval.base_type, pytd.ClassType), pyval base = pyval.base_type.cls assert isinstance(base, pytd.Class), base base_cls = self.constant_to_value(base, subst, self.ctx.root_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = list(range(len(pyval.parameters))) + [abstract_utils.T] combined_parameter = pytd_utils.JoinTypes(pyval.parameters) parameters = pyval.parameters + (combined_parameter,) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.CallableClass template = list(range(len(pyval.args))) + [abstract_utils.ARGS, abstract_utils.RET] parameters = pyval.args + (pytd_utils.JoinTypes(pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass if pyval.name == "typing.Generic": pyval_template = pyval.parameters else: pyval_template = base.template template = tuple(t.name for t in pyval_template) parameters = pyval.parameters assert (pyval.name in ("typing.Generic", "typing.Protocol") or len(parameters) <= len(template)) # Delay type parameter loading to handle recursive types. # See the ParameterizedClass.formal_type_parameters() property. type_parameters = abstract_utils.LazyFormalTypeParameters( template, parameters, subst) return abstract_class(base_cls, type_parameters, self.ctx) elif isinstance(pyval, pytd.Literal): value = self.constant_to_value( self._get_literal_value(pyval.value), subst, self.ctx.root_node) return abstract.LiteralClass(value, self.ctx) elif isinstance(pyval, pytd.Annotated): typ = self.constant_to_value(pyval.base_type, subst, self.ctx.root_node) if pyval.annotations[0] == "'pytype_metadata'": try: md = metadata.from_string(pyval.annotations[1]) if md["tag"] == "attr.ib": ret = attr_overlay.AttribInstance.from_metadata( self.ctx, self.ctx.root_node, typ, md) return ret elif md["tag"] == "attr.s": ret = attr_overlay.Attrs.from_metadata(self.ctx, md) return ret except (IndexError, ValueError, TypeError, KeyError): details = "Wrong format for pytype_metadata." self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames, pyval.annotations[1], details) return typ else: return typ elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.ctx.root_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))