def test_union_equality(self): union1 = abstract.Union((self._ctx.convert.unsolvable,), self._ctx) union2 = abstract.Union((self._ctx.convert.none,), self._ctx) cls = abstract.InterpreterClass("Union", [], {}, None, self._ctx) self.assertEqual(union1, union1) self.assertNotEqual(union1, union2) self.assertNotEqual(union1, cls)
def test_heterogeneous_tuple(self): ast = self._load_ast("a", """ from typing import Tuple x = ... # type: Tuple[str, int] """) x = ast.Lookup("a.x").type cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node) instance = self._ctx.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._ctx.root_node) self.assertIsInstance(cls, abstract.TupleClass) self.assertCountEqual(cls.formal_type_parameters.items(), [(0, self._ctx.convert.str_type), (1, self._ctx.convert.int_type), (abstract_utils.T, abstract.Union([ cls.formal_type_parameters[0], cls.formal_type_parameters[1], ], self._ctx))]) self.assertIsInstance(instance, abstract.Tuple) self.assertListEqual([v.data for v in instance.pyval], [[self._ctx.convert.primitive_class_instances[str]], [self._ctx.convert.primitive_class_instances[int]]]) # The order of option elements in Union is random self.assertCountEqual( instance.get_instance_type_parameter(abstract_utils.T).data, [ self._ctx.convert.primitive_class_instances[str], self._ctx.convert.primitive_class_instances[int] ])
def test_callable_with_args(self): ast = self._load_ast("a", """ from typing import Callable x = ... # type: Callable[[int, bool], str] """) x = ast.Lookup("a.x").type cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node) instance = self._ctx.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._ctx.root_node) self.assertIsInstance(cls, abstract.CallableClass) self.assertCountEqual( cls.formal_type_parameters.items(), [(0, self._ctx.convert.int_type), (1, self._ctx.convert.primitive_classes[bool]), (abstract_utils.ARGS, abstract.Union( [cls.formal_type_parameters[0], cls.formal_type_parameters[1]], self._ctx)), (abstract_utils.RET, self._ctx.convert.str_type)]) self.assertIsInstance(instance, abstract.Instance) self.assertEqual(instance.cls, cls) self.assertCountEqual( [(name, set(var.data)) for name, var in instance.instance_type_parameters.items()], [(abstract_utils.full_type_name(instance, abstract_utils.ARGS), { self._ctx.convert.primitive_class_instances[int], self._ctx.convert.primitive_class_instances[bool] }), (abstract_utils.full_type_name(instance, abstract_utils.RET), {self._ctx.convert.primitive_class_instances[str]})])
def _build_value(self, node, inner, ellipses): self.ctx.errorlog.invalid_ellipses(self.ctx.vm.frames, ellipses, self.name) if len(inner) != 1: error = "typing.Optional can only contain one type parameter" self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames, self, error) return abstract.Union((self.ctx.convert.none_type, ) + inner, self.ctx)
def merge_values(self, values): """Merge a collection of values into a single one.""" if not values: return self.empty elif len(values) == 1: return next(iter(values)) else: return abstract.Union(values, self.ctx)
def test_metaclass_union(self): cls = abstract.InterpreterClass("X", [], {}, None, self._ctx) meta1 = abstract.InterpreterClass("M1", [], {}, None, self._ctx) meta2 = abstract.InterpreterClass("M2", [], {}, None, self._ctx) meta1.official_name = "M1" meta2.official_name = "M2" cls.cls = abstract.Union([meta1, meta2], self._ctx) pytd_cls = cls.to_pytd_def(self._ctx.root_node, "X") self.assertEqual(pytd_cls.metaclass, pytd.UnionType( (pytd.NamedType("M1"), pytd.NamedType("M2"))))
def test_union_set_attribute(self): list_instance = abstract.Instance(self._ctx.convert.list_type, self._ctx) cls = abstract.InterpreterClass("obj", [], {}, None, self._ctx) cls_instance = abstract.Instance(cls, self._ctx) union = abstract.Union([cls_instance, list_instance], self._ctx) node = self._ctx.attribute_handler.set_attribute( self._ctx.root_node, union, "rumpelstiltskin", self._ctx.convert.none_type.to_variable(self._ctx.root_node)) self.assertEqual(cls_instance.members["rumpelstiltskin"].data.pop(), self._ctx.convert.none_type) self.assertIs(node, self._ctx.root_node) error, = self._ctx.errorlog.unique_sorted_errors() self.assertEqual(error.name, "not-writable")
def _process_annotation(param): """Process a single param into annotations.""" if not param.typ: return elif isinstance(param.typ, cfg.Variable): types = param.typ.data if len(types) == 1: annotations[param.name] = types[0].cls else: t = abstract.Union([t.cls for t in types], ctx) annotations[param.name] = t else: annotations[param.name] = param.typ
def test_signature_from_callable(self): # Callable[[int, str], Any] params = {0: self._ctx.convert.int_type, 1: self._ctx.convert.str_type} params[abstract_utils.ARGS] = abstract.Union((params[0], params[1]), self._ctx) params[abstract_utils.RET] = self._ctx.convert.unsolvable callable_val = abstract.CallableClass(self._ctx.convert.function_type, params, self._ctx) sig = function.Signature.from_callable(callable_val) self.assertEqual(repr(sig), "def <callable>(_0: int, _1: str) -> Any") self.assertEqual(sig.name, "<callable>") self.assertSequenceEqual(sig.param_names, ("_0", "_1")) self.assertIs(sig.varargs_name, None) self.assertFalse(sig.kwonly_params) self.assertIs(sig.kwargs_name, None) self.assertCountEqual(sig.annotations.keys(), sig.param_names + ("return",)) self.assertTrue(sig.has_return_annotation) self.assertTrue(sig.has_param_annotations)
def _build_value(self, node, inner, ellipses): self.ctx.errorlog.invalid_ellipses(self.ctx.vm.frames, ellipses, self.name) return abstract.Union(self.options + inner, self.ctx)
def _constant_to_value(self, pyval, subst, get_node): """Create a BaseValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if isinstance(pyval, str): return abstract.ConcreteValue(pyval, self.str_type, self.ctx) elif isinstance(pyval, bytes): return abstract.ConcreteValue(pyval, self.bytes_type, self.ctx) elif isinstance(pyval, bool): return self.true if pyval else self.false elif isinstance(pyval, int) and -1 <= pyval <= _MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.ConcreteValue(pyval, self.int_type, self.ctx) elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif pyval.__class__ is frozenset: instance = abstract.Instance(self.frozenset_type, self.ctx) for element in pyval: instance.merge_instance_type_parameter( self.ctx.root_node, abstract_utils.T, self.constant_to_var(element, subst, self.ctx.root_node)) return instance elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.ConcreteValue(pyval, self.primitive_classes[types.CodeType], self.ctx) elif pyval is super: return special_builtins.Super(self.ctx) elif pyval is object: return special_builtins.Object(self.ctx) elif pyval.__class__ is type: try: return self.name_to_value(self._type_to_name(pyval), subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.LateType): actual = self._load_late_type(pyval) return self._constant_to_value(actual, subst, get_node) elif isinstance(pyval, pytd.TypeDeclUnit): return self._create_module(pyval) elif isinstance(pyval, pytd.Module): mod = self.ctx.loader.import_name(pyval.module_name) return self._create_module(mod) elif isinstance(pyval, pytd.Class): if pyval.name == "builtins.super": return self.ctx.special_builtins["super"] elif pyval.name == "builtins.object": return self.object_type elif pyval.name == "types.ModuleType": return self.module_type elif pyval.name == "_importlib_modulespec.ModuleType": # Python 3's typeshed uses a stub file indirection to define ModuleType # even though it is exported via types.pyi. return self.module_type elif pyval.name == "types.FunctionType": return self.function_type else: module, dot, base_name = pyval.name.rpartition(".") # typing.TypingContainer intentionally loads the underlying pytd types. if (module not in ("typing", "typing_extensions") and module in overlay_dict.overlays): overlay = self.ctx.vm.import_module(module, module, 0) if overlay.get_module(base_name) is overlay: overlay.load_lazy_attribute(base_name) return abstract_utils.get_atomic_value(overlay.members[base_name]) try: cls = abstract.PyTDClass.make(base_name, pyval, self.ctx) except mro.MROError as e: self.ctx.errorlog.mro_error(self.ctx.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module cls.call_metaclass_init(get_node()) return cls elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.ctx) for sig in pyval.signatures ] type_new = self.ctx.loader.lookup_builtin("builtins.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.ctx) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): if pyval.cls: cls = pyval.cls else: # If pyval is a reference to a class in builtins or typing, we can fill # in the class ourselves. lookup_builtin raises a KeyError if the name # is not found. cls = self.ctx.loader.lookup_builtin(pyval.name) assert isinstance(cls, pytd.Class) return self.constant_to_value(cls, subst, self.ctx.root_node) elif isinstance(pyval, pytd.NothingType): return self.empty elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.AnythingType)): # We allow "X = ... # type: Any" to declare X as a type. return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.GenericType) and pyval.type.name == "builtins.type"): # `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`. param, = pyval.type.parameters return self.constant_to_value(param, subst, self.ctx.root_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.ctx.root_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.ctx) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.ctx.root_node) for c in pyval.constraints) bound = ( pyval.bound and self.constant_to_value(pyval.bound, {}, self.ctx.root_node)) return abstract.TypeParameter( pyval.name, self.ctx, constraints=constraints, bound=bound, module=pyval.scope) elif isinstance(pyval, abstract_utils.AsInstance): cls = pyval.cls if isinstance(cls, pytd.LateType): actual = self._load_late_type(cls) if not isinstance(actual, pytd.ClassType): return self.unsolvable cls = actual.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar": param, = cls.parameters return self.constant_to_value( abstract_utils.AsInstance(param), subst, self.ctx.root_node) elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and cls.template): # If we're converting a generic Class, need to create a new instance of # it. See test_classes.testGenericReinstantiated. if isinstance(cls, pytd.Class): params = tuple(t.type_param.upper_value for t in cls.template) cls = pytd.GenericType(base_type=pytd.ClassType(cls.name, cls), parameters=params) if isinstance(cls.base_type, pytd.LateType): actual = self._load_late_type(cls.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base_cls = actual.cls else: base_type = cls.base_type assert isinstance(base_type, pytd.ClassType) base_cls = base_type.cls assert isinstance(base_cls, pytd.Class), base_cls if base_cls.name == "builtins.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.full_name not in subst: raise self.TypeParameterError(c.full_name) # deformalize gets rid of any unexpected TypeVars, which can appear # if something is annotated as Type[T]. return self.ctx.annotation_utils.deformalize( self.merge_classes(subst[c.full_name].data)) else: return self.constant_to_value(c, subst, self.ctx.root_node) elif isinstance(cls, pytd.TupleType): content = tuple(self.constant_to_var(abstract_utils.AsInstance(p), subst, get_node()) for p in cls.parameters) return self.tuple_to_value(content) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.ctx.root_node) return abstract.Instance(clsval, self.ctx) else: clsval = self.constant_to_value(base_cls, subst, self.ctx.root_node) instance = abstract.Instance(clsval, self.ctx) num_params = len(cls.parameters) assert num_params <= len(base_cls.template) for i, formal in enumerate(base_cls.template): if i < num_params: node = get_node() p = self.constant_to_var( abstract_utils.AsInstance(cls.parameters[i]), subst, node) else: # An omitted type parameter implies `Any`. node = self.ctx.root_node p = self.unsolvable.to_variable(node) instance.merge_instance_type_parameter(node, formal.name, p) return instance elif isinstance(cls, pytd.Class): assert not cls.template # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in ["builtins.type", "builtins.property"]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.ctx.root_node) instance = abstract.Instance(mycls, self.ctx) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] elif isinstance(cls, pytd.Literal): return self.constant_to_value( self._get_literal_value(cls.value), subst, self.ctx.root_node) else: return self.constant_to_value(cls, subst, self.ctx.root_node) elif (isinstance(pyval, pytd.GenericType) and pyval.name == "typing.ClassVar"): param, = pyval.parameters return self.constant_to_value(param, subst, self.ctx.root_node) elif isinstance(pyval, pytd.GenericType): if isinstance(pyval.base_type, pytd.LateType): actual = self._load_late_type(pyval.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base = actual.cls else: assert isinstance(pyval.base_type, pytd.ClassType), pyval base = pyval.base_type.cls assert isinstance(base, pytd.Class), base base_cls = self.constant_to_value(base, subst, self.ctx.root_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = list(range(len(pyval.parameters))) + [abstract_utils.T] combined_parameter = pytd_utils.JoinTypes(pyval.parameters) parameters = pyval.parameters + (combined_parameter,) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.CallableClass template = list(range(len(pyval.args))) + [abstract_utils.ARGS, abstract_utils.RET] parameters = pyval.args + (pytd_utils.JoinTypes(pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass if pyval.name == "typing.Generic": pyval_template = pyval.parameters else: pyval_template = base.template template = tuple(t.name for t in pyval_template) parameters = pyval.parameters assert (pyval.name in ("typing.Generic", "typing.Protocol") or len(parameters) <= len(template)) # Delay type parameter loading to handle recursive types. # See the ParameterizedClass.formal_type_parameters() property. type_parameters = abstract_utils.LazyFormalTypeParameters( template, parameters, subst) return abstract_class(base_cls, type_parameters, self.ctx) elif isinstance(pyval, pytd.Literal): value = self.constant_to_value( self._get_literal_value(pyval.value), subst, self.ctx.root_node) return abstract.LiteralClass(value, self.ctx) elif isinstance(pyval, pytd.Annotated): typ = self.constant_to_value(pyval.base_type, subst, self.ctx.root_node) if pyval.annotations[0] == "'pytype_metadata'": try: md = metadata.from_string(pyval.annotations[1]) if md["tag"] == "attr.ib": ret = attr_overlay.AttribInstance.from_metadata( self.ctx, self.ctx.root_node, typ, md) return ret elif md["tag"] == "attr.s": ret = attr_overlay.Attrs.from_metadata(self.ctx, md) return ret except (IndexError, ValueError, TypeError, KeyError): details = "Wrong format for pytype_metadata." self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames, pyval.annotations[1], details) return typ else: return typ elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.ctx.root_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def _build_namedtuple(self, name, field_names, field_types, node, bases): # Build an InterpreterClass representing the namedtuple. if field_types: # TODO(mdemello): Fix this to support late types. field_types_union = abstract.Union(field_types, self.ctx) else: field_types_union = self.ctx.convert.none_type members = {n: t.instantiate(node) for n, t in zip(field_names, field_types)} # collections.namedtuple has: __dict__, __slots__ and _fields. # typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults. # __slots__ and _fields are tuples containing the names of the fields. slots = tuple(self.ctx.convert.build_string(node, f) for f in field_names) members["__slots__"] = self.ctx.convert.build_tuple(node, slots) members["_fields"] = self.ctx.convert.build_tuple(node, slots) # __dict__ and _field_defaults are both collections.OrderedDicts that map # field names (strings) to objects of the field types. ordered_dict_cls = self.ctx.convert.name_to_value( "collections.OrderedDict", ast=self.collections_ast) # Normally, we would use abstract_utils.K and abstract_utils.V, but # collections.pyi doesn't conform to that standard. field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, { "K": self.ctx.convert.str_type, "V": field_types_union }, self.ctx) members["__dict__"] = field_dict_cls.instantiate(node) members["_field_defaults"] = field_dict_cls.instantiate(node) # _field_types and __annotations__ are both collections.OrderedDicts # that map field names (strings) to the types of the fields. Note that # ctx.make_class will take care of adding the __annotations__ member. field_types_cls = abstract.ParameterizedClass(ordered_dict_cls, { "K": self.ctx.convert.str_type, "V": self.ctx.convert.type_type }, self.ctx) members["_field_types"] = field_types_cls.instantiate(node) # __new__ # We set the bound on this TypeParameter later. This gives __new__ the # signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same # signature that visitor.CreateTypeParametersForSignatures would create. # This allows subclasses of the NamedTuple to get the correct type from # their constructors. cls_type_param = abstract.TypeParameter( visitors.CreateTypeParametersForSignatures.PREFIX + name, self.ctx, bound=None) cls_type = abstract.ParameterizedClass(self.ctx.convert.type_type, {abstract_utils.T: cls_type_param}, self.ctx) params = [Param(n, t) for n, t in zip(field_names, field_types)] members["__new__"] = overlay_utils.make_method( self.ctx, node, name="__new__", self_param=Param("cls", cls_type), params=params, return_type=cls_type_param, ) # __init__ members["__init__"] = overlay_utils.make_method( self.ctx, node, name="__init__", varargs=Param("args"), kwargs=Param("kwargs")) heterogeneous_tuple_type_params = dict(enumerate(field_types)) heterogeneous_tuple_type_params[abstract_utils.T] = field_types_union # Representation of the to-be-created NamedTuple as a typing.Tuple. heterogeneous_tuple_type = abstract.TupleClass( self.ctx.convert.tuple_type, heterogeneous_tuple_type_params, self.ctx) # _make # _make is a classmethod, so it needs to be wrapped by # special_builtins.ClassMethodInstance. # Like __new__, it uses the _Tname TypeVar. sized_cls = self.ctx.convert.name_to_value("typing.Sized") iterable_type = abstract.ParameterizedClass( self.ctx.convert.name_to_value("typing.Iterable"), {abstract_utils.T: field_types_union}, self.ctx) cls_type = abstract.ParameterizedClass(self.ctx.convert.type_type, {abstract_utils.T: cls_type_param}, self.ctx) len_type = abstract.CallableClass( self.ctx.convert.name_to_value("typing.Callable"), { 0: sized_cls, abstract_utils.ARGS: sized_cls, abstract_utils.RET: self.ctx.convert.int_type }, self.ctx) params = [ Param("iterable", iterable_type), Param("new").unsolvable(self.ctx, node), Param("len", len_type).unsolvable(self.ctx, node) ] make = overlay_utils.make_method( self.ctx, node, name="_make", params=params, self_param=Param("cls", cls_type), return_type=cls_type_param) make_args = function.Args(posargs=(make,)) _, members["_make"] = self.ctx.special_builtins["classmethod"].call( node, None, make_args) # _replace # Like __new__, it uses the _Tname TypeVar. We have to annotate the `self` # param to make sure the TypeVar is substituted correctly. members["_replace"] = overlay_utils.make_method( self.ctx, node, name="_replace", self_param=Param("self", cls_type_param), return_type=cls_type_param, kwargs=Param("kwds", field_types_union)) # __getnewargs__ members["__getnewargs__"] = overlay_utils.make_method( self.ctx, node, name="__getnewargs__", return_type=heterogeneous_tuple_type) # __getstate__ members["__getstate__"] = overlay_utils.make_method( self.ctx, node, name="__getstate__") # _asdict members["_asdict"] = overlay_utils.make_method( self.ctx, node, name="_asdict", return_type=field_dict_cls) # Finally, make the class. cls_dict = abstract.Dict(self.ctx) cls_dict.update(node, members) if self.ctx.options.strict_namedtuple_checks: # Enforces type checking like Tuple[...] superclass_of_new_type = heterogeneous_tuple_type.to_variable(node) else: superclass_of_new_type = self.ctx.convert.tuple_type.to_variable(node) if bases: final_bases = [] for base in bases: if any(b.full_name == "typing.NamedTuple" for b in base.data): final_bases.append(superclass_of_new_type) else: final_bases.append(base) else: final_bases = [superclass_of_new_type] # This NamedTuple is being created via a function call. We manually # construct an annotated_locals entry for it so that __annotations__ is # initialized properly for the generated class. self.ctx.vm.annotated_locals[name] = { field: abstract_utils.Local(node, None, typ, None, self.ctx) for field, typ in zip(field_names, field_types) } node, cls_var = self.ctx.make_class( node=node, name_var=self.ctx.convert.build_string(node, name), bases=final_bases, class_dict_var=cls_dict.to_variable(node), cls_var=None) cls = cls_var.data[0] # Now that the class has been made, we can complete the TypeParameter used # by __new__, _make and _replace. cls_type_param.bound = cls return node, cls_var