def test_union_set_attribute(self): list_instance = abstract.Instance(self._ctx.convert.list_type, self._ctx) cls = abstract.InterpreterClass("obj", [], {}, None, self._ctx) cls_instance = abstract.Instance(cls, self._ctx) union = abstract.Union([cls_instance, list_instance], self._ctx) node = self._ctx.attribute_handler.set_attribute( self._ctx.root_node, union, "rumpelstiltskin", self._ctx.convert.none_type.to_variable(self._ctx.root_node)) self.assertEqual(cls_instance.members["rumpelstiltskin"].data.pop(), self._ctx.convert.none_type) self.assertIs(node, self._ctx.root_node) error, = self._ctx.errorlog.unique_sorted_errors() self.assertEqual(error.name, "not-writable")
def build_set(self, node, content): """Create a VM set from the given sequence.""" content = list(content) # content might be a generator value = abstract.Instance(self.set_type, self.ctx) value.merge_instance_type_parameter( node, abstract_utils.T, self.build_content(content)) return value.to_variable(node)
def test_getitem_with_instance_valself(self): cls = abstract.InterpreterClass("X", [], {}, None, self.ctx) valself = abstract.Instance(cls, self.ctx).to_binding(self.node) _, attr_var = self.attribute_handler.get_attribute( self.node, cls, "__getitem__", valself) # Since we passed in `valself` for this lookup of __getitem__ on a class, # it is treated as a normal lookup; X.__getitem__ does not exist. self.assertIsNone(attr_var)
def create_kwargs(self, node): key_type = self.ctx.convert.primitive_class_instances[str].to_variable( node) value_type = self.ctx.convert.create_new_unknown(node) kwargs = abstract.Instance(self.ctx.convert.dict_type, self.ctx) kwargs.merge_instance_type_parameter(node, abstract_utils.K, key_type) kwargs.merge_instance_type_parameter(node, abstract_utils.V, value_type) return kwargs.to_variable(node)
def unpack_and_build(state, count, build_concrete, container_type, ctx): state, seq = pop_and_unpack_list(state, count, ctx) if any(abstract_utils.is_var_splat(x) for x in seq): retval = abstract.Instance(container_type, ctx) merge_indefinite_iterables(state.node, retval, seq) ret = retval.to_variable(state.node) else: ret = build_concrete(state.node, seq) return state.push(ret)
def test_instance_no_valself(self): instance = abstract.Instance(self.ctx.convert.int_type, self.ctx) _, attr_var = self.attribute_handler.get_attribute( self.node, instance, "real") attr_binding, = attr_var.bindings self.assertEqual(attr_binding.data.cls, self.ctx.convert.int_type) # Since `valself` was not passed to get_attribute, a binding to # `instance` is not among the attribute's origins. self.assertNotIn(instance, [o.data for o in _get_origins(attr_binding)])
def test_call_empty_type_parameter_instance(self): instance = abstract.Instance(self._ctx.convert.list_type, self._ctx) t = abstract.TypeParameter(abstract_utils.T, self._ctx) t_instance = abstract.TypeParameterInstance(t, instance, self._ctx) node, ret = t_instance.call(self._node, t_instance.to_binding(self._node), function.Args(posargs=())) self.assertIs(node, self._node) retval, = ret.data self.assertIs(retval, self._ctx.convert.empty)
def test_compatible_with_set(self): i = abstract.Instance(self._convert.set_type, self._ctx) # Empty set is not compatible with True. self.assertFalsy(i) # Once a type parameter is set, list is compatible with True and False. i.merge_instance_type_parameter( self._node, abstract_utils.T, self._convert.object_type.to_variable(self._ctx.root_node)) self.assertAmbiguous(i)
def test_instance_with_valself(self): instance = abstract.Instance(self.ctx.convert.int_type, self.ctx) valself = instance.to_binding(self.node) _, attr_var = self.attribute_handler.get_attribute( self.node, instance, "real", valself) attr_binding, = attr_var.bindings self.assertEqual(attr_binding.data.cls, self.ctx.convert.int_type) # Since `valself` was passed to get_attribute, it is added to the # attribute's origins. self.assertIn(valself, _get_origins(attr_binding))
def test_empty_type_parameter_instance(self): t = abstract.TypeParameter( abstract_utils.T, self._ctx, bound=self._ctx.convert.int_type) instance = abstract.Instance(self._ctx.convert.list_type, self._ctx) t_instance = abstract.TypeParameterInstance(t, instance, self._ctx) node, var = self._ctx.attribute_handler.get_attribute( self._ctx.root_node, t_instance, "real") self.assertIs(node, self._ctx.root_node) attr, = var.data self.assertIs(attr, self._ctx.convert.primitive_class_instances[int])
def test_class_with_instance_valself(self): meta_members = {"x": self.ctx.convert.none.to_variable(self.node)} meta = abstract.InterpreterClass("M", [], meta_members, None, self.ctx) cls = abstract.InterpreterClass("X", [], {}, meta, self.ctx) valself = abstract.Instance(cls, self.ctx).to_binding(self.node) _, attr_var = self.attribute_handler.get_attribute( self.node, cls, "x", valself) # Since `valself` is an instance of X, we do not look at the metaclass, so # M.x is not returned. self.assertIsNone(attr_var)
def test_call_type_parameter_instance(self): instance = abstract.Instance(self._ctx.convert.list_type, self._ctx) instance.merge_instance_type_parameter( self._ctx.root_node, abstract_utils.T, self._ctx.convert.int_type.to_variable(self._ctx.root_node)) t = abstract.TypeParameter(abstract_utils.T, self._ctx) t_instance = abstract.TypeParameterInstance(t, instance, self._ctx) node, ret = t_instance.call(self._node, t_instance.to_binding(self._node), function.Args(posargs=())) self.assertIs(node, self._node) retval, = ret.data self.assertEqual(retval.cls, self._ctx.convert.int_type)
def test_call_type_parameter_instance_with_wrong_args(self): instance = abstract.Instance(self._ctx.convert.list_type, self._ctx) instance.merge_instance_type_parameter( self._ctx.root_node, abstract_utils.T, self._ctx.convert.int_type.to_variable(self._ctx.root_node)) t = abstract.TypeParameter(abstract_utils.T, self._ctx) t_instance = abstract.TypeParameterInstance(t, instance, self._ctx) posargs = (self._ctx.new_unsolvable(self._node),) * 3 node, ret = t_instance.call(self._node, t_instance.to_binding(self._node), function.Args(posargs=posargs)) self.assertIs(node, self._node) self.assertTrue(ret.bindings) error, = self._ctx.errorlog self.assertEqual(error.name, "wrong-arg-count")
def build_collection_of_type(self, node, typ, var): """Create a collection Typ[T] with T derived from the given variable.""" ret = abstract.Instance(typ, self.ctx) ret.merge_instance_type_parameter(node, abstract_utils.T, var) return ret.to_variable(node)
def test_compare_frozensets(self): """Test that two frozensets can be compared for equality.""" fset = self._convert.frozenset_type i = abstract.Instance(fset, self._ctx) j = abstract.Instance(fset, self._ctx) self.assertIs(None, compare.cmp_rel(self._ctx, slots.EQ, i, j))
def test_compatible_with_none(self): # This test is specifically for abstract.Instance, so we don't use # self._convert.none, which is a ConcreteValue. i = abstract.Instance(self._convert.none_type, self._ctx) self.assertFalsy(i)
def __init__(self, ctx): super().__init__(ctx) ctx.convert = self # to make constant_to_value calls below work self._convert_cache = {} self._resolved_late_types = {} # performance cache # Initialize primitive_classes to empty to allow constant_to_value to run. self.primitive_classes = () # object_type is needed to initialize the primitive class values. self.object_type = self.constant_to_value(object) self.unsolvable = abstract.Unsolvable(self.ctx) self.type_type = self.constant_to_value(type) self.ctx.converter_minimally_initialized = True self.empty = abstract.Empty(self.ctx) self.no_return = typing_overlay.NoReturn(self.ctx) # Now fill primitive_classes with the real values using constant_to_value. primitive_classes = [ int, float, str, bytes, object, NoneType, complex, bool, slice, types.CodeType, EllipsisType, super, ] self.primitive_classes = { v: self.constant_to_value(v) for v in primitive_classes } self.primitive_class_names = [ self._type_to_name(x) for x in self.primitive_classes] self.none = abstract.ConcreteValue(None, self.primitive_classes[NoneType], self.ctx) self.true = abstract.ConcreteValue(True, self.primitive_classes[bool], self.ctx) self.false = abstract.ConcreteValue(False, self.primitive_classes[bool], self.ctx) self.ellipsis = abstract.ConcreteValue(Ellipsis, self.primitive_classes[EllipsisType], self.ctx) self.primitive_class_instances = {} for name, cls in self.primitive_classes.items(): if name == NoneType: # This is possible because all None instances are the same. # Without it pytype could not reason that "x is None" is always true, if # x is indeed None. instance = self.none elif name == EllipsisType: instance = self.ellipsis else: instance = abstract.Instance(cls, self.ctx) self.primitive_class_instances[name] = instance self._convert_cache[(abstract.Instance, cls.pytd_cls)] = instance self.none_type = self.primitive_classes[NoneType] self.super_type = self.primitive_classes[super] self.str_type = self.primitive_classes[str] self.int_type = self.primitive_classes[int] self.bool_type = self.primitive_classes[bool] self.bytes_type = self.primitive_classes[bytes] self.list_type = self.constant_to_value(list) self.set_type = self.constant_to_value(set) self.frozenset_type = self.constant_to_value(frozenset) self.dict_type = self.constant_to_value(dict) self.module_type = self.constant_to_value(types.ModuleType) self.function_type = self.constant_to_value(types.FunctionType) self.tuple_type = self.constant_to_value(tuple) self.generator_type = self.constant_to_value(types.GeneratorType) self.iterator_type = self.constant_to_value(IteratorType) self.coroutine_type = self.constant_to_value(CoroutineType) self.awaitable_type = self.constant_to_value(AwaitableType) self.async_generator_type = self.constant_to_value(AsyncGeneratorType) self.bool_values = { True: self.true, False: self.false, None: self.primitive_class_instances[bool], }
def test_compatible_with_numeric(self): # Numbers can evaluate to True or False i = abstract.Instance(self._convert.int_type, self._ctx) self.assertAmbiguous(i)
def create_varargs(self, node): value = abstract.Instance(self.ctx.convert.tuple_type, self.ctx) value.merge_instance_type_parameter( node, abstract_utils.T, self.ctx.convert.create_new_unknown(node)) return value.to_variable(node)
def test_cmp_rel__unknown(self): tup1 = self._convert.constant_to_value((3, 1)) tup2 = abstract.Instance(self._convert.tuple_type, self._ctx) for op in (slots.LT, slots.LE, slots.EQ, slots.NE, slots.GE, slots.GT): self.assertIsNone(compare.cmp_rel(self._ctx, op, tup1, tup2)) self.assertIsNone(compare.cmp_rel(self._ctx, op, tup2, tup1))
def test_compatible_with_object(self): # object() is not compatible with False i = abstract.Instance(self._convert.object_type, self._ctx) self.assertTruthy(i)
def _constant_to_value(self, pyval, subst, get_node): """Create a BaseValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if isinstance(pyval, str): return abstract.ConcreteValue(pyval, self.str_type, self.ctx) elif isinstance(pyval, bytes): return abstract.ConcreteValue(pyval, self.bytes_type, self.ctx) elif isinstance(pyval, bool): return self.true if pyval else self.false elif isinstance(pyval, int) and -1 <= pyval <= _MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.ConcreteValue(pyval, self.int_type, self.ctx) elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif pyval.__class__ is frozenset: instance = abstract.Instance(self.frozenset_type, self.ctx) for element in pyval: instance.merge_instance_type_parameter( self.ctx.root_node, abstract_utils.T, self.constant_to_var(element, subst, self.ctx.root_node)) return instance elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.ConcreteValue(pyval, self.primitive_classes[types.CodeType], self.ctx) elif pyval is super: return special_builtins.Super(self.ctx) elif pyval is object: return special_builtins.Object(self.ctx) elif pyval.__class__ is type: try: return self.name_to_value(self._type_to_name(pyval), subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.LateType): actual = self._load_late_type(pyval) return self._constant_to_value(actual, subst, get_node) elif isinstance(pyval, pytd.TypeDeclUnit): return self._create_module(pyval) elif isinstance(pyval, pytd.Module): mod = self.ctx.loader.import_name(pyval.module_name) return self._create_module(mod) elif isinstance(pyval, pytd.Class): if pyval.name == "builtins.super": return self.ctx.special_builtins["super"] elif pyval.name == "builtins.object": return self.object_type elif pyval.name == "types.ModuleType": return self.module_type elif pyval.name == "_importlib_modulespec.ModuleType": # Python 3's typeshed uses a stub file indirection to define ModuleType # even though it is exported via types.pyi. return self.module_type elif pyval.name == "types.FunctionType": return self.function_type else: module, dot, base_name = pyval.name.rpartition(".") # typing.TypingContainer intentionally loads the underlying pytd types. if (module not in ("typing", "typing_extensions") and module in overlay_dict.overlays): overlay = self.ctx.vm.import_module(module, module, 0) if overlay.get_module(base_name) is overlay: overlay.load_lazy_attribute(base_name) return abstract_utils.get_atomic_value(overlay.members[base_name]) try: cls = abstract.PyTDClass.make(base_name, pyval, self.ctx) except mro.MROError as e: self.ctx.errorlog.mro_error(self.ctx.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module cls.call_metaclass_init(get_node()) return cls elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.ctx) for sig in pyval.signatures ] type_new = self.ctx.loader.lookup_builtin("builtins.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.ctx) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): if pyval.cls: cls = pyval.cls else: # If pyval is a reference to a class in builtins or typing, we can fill # in the class ourselves. lookup_builtin raises a KeyError if the name # is not found. cls = self.ctx.loader.lookup_builtin(pyval.name) assert isinstance(cls, pytd.Class) return self.constant_to_value(cls, subst, self.ctx.root_node) elif isinstance(pyval, pytd.NothingType): return self.empty elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.AnythingType)): # We allow "X = ... # type: Any" to declare X as a type. return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.GenericType) and pyval.type.name == "builtins.type"): # `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`. param, = pyval.type.parameters return self.constant_to_value(param, subst, self.ctx.root_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.ctx.root_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.ctx) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.ctx.root_node) for c in pyval.constraints) bound = ( pyval.bound and self.constant_to_value(pyval.bound, {}, self.ctx.root_node)) return abstract.TypeParameter( pyval.name, self.ctx, constraints=constraints, bound=bound, module=pyval.scope) elif isinstance(pyval, abstract_utils.AsInstance): cls = pyval.cls if isinstance(cls, pytd.LateType): actual = self._load_late_type(cls) if not isinstance(actual, pytd.ClassType): return self.unsolvable cls = actual.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar": param, = cls.parameters return self.constant_to_value( abstract_utils.AsInstance(param), subst, self.ctx.root_node) elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and cls.template): # If we're converting a generic Class, need to create a new instance of # it. See test_classes.testGenericReinstantiated. if isinstance(cls, pytd.Class): params = tuple(t.type_param.upper_value for t in cls.template) cls = pytd.GenericType(base_type=pytd.ClassType(cls.name, cls), parameters=params) if isinstance(cls.base_type, pytd.LateType): actual = self._load_late_type(cls.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base_cls = actual.cls else: base_type = cls.base_type assert isinstance(base_type, pytd.ClassType) base_cls = base_type.cls assert isinstance(base_cls, pytd.Class), base_cls if base_cls.name == "builtins.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.full_name not in subst: raise self.TypeParameterError(c.full_name) # deformalize gets rid of any unexpected TypeVars, which can appear # if something is annotated as Type[T]. return self.ctx.annotation_utils.deformalize( self.merge_classes(subst[c.full_name].data)) else: return self.constant_to_value(c, subst, self.ctx.root_node) elif isinstance(cls, pytd.TupleType): content = tuple(self.constant_to_var(abstract_utils.AsInstance(p), subst, get_node()) for p in cls.parameters) return self.tuple_to_value(content) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.ctx.root_node) return abstract.Instance(clsval, self.ctx) else: clsval = self.constant_to_value(base_cls, subst, self.ctx.root_node) instance = abstract.Instance(clsval, self.ctx) num_params = len(cls.parameters) assert num_params <= len(base_cls.template) for i, formal in enumerate(base_cls.template): if i < num_params: node = get_node() p = self.constant_to_var( abstract_utils.AsInstance(cls.parameters[i]), subst, node) else: # An omitted type parameter implies `Any`. node = self.ctx.root_node p = self.unsolvable.to_variable(node) instance.merge_instance_type_parameter(node, formal.name, p) return instance elif isinstance(cls, pytd.Class): assert not cls.template # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in ["builtins.type", "builtins.property"]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.ctx.root_node) instance = abstract.Instance(mycls, self.ctx) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] elif isinstance(cls, pytd.Literal): return self.constant_to_value( self._get_literal_value(cls.value), subst, self.ctx.root_node) else: return self.constant_to_value(cls, subst, self.ctx.root_node) elif (isinstance(pyval, pytd.GenericType) and pyval.name == "typing.ClassVar"): param, = pyval.parameters return self.constant_to_value(param, subst, self.ctx.root_node) elif isinstance(pyval, pytd.GenericType): if isinstance(pyval.base_type, pytd.LateType): actual = self._load_late_type(pyval.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base = actual.cls else: assert isinstance(pyval.base_type, pytd.ClassType), pyval base = pyval.base_type.cls assert isinstance(base, pytd.Class), base base_cls = self.constant_to_value(base, subst, self.ctx.root_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = list(range(len(pyval.parameters))) + [abstract_utils.T] combined_parameter = pytd_utils.JoinTypes(pyval.parameters) parameters = pyval.parameters + (combined_parameter,) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.CallableClass template = list(range(len(pyval.args))) + [abstract_utils.ARGS, abstract_utils.RET] parameters = pyval.args + (pytd_utils.JoinTypes(pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass if pyval.name == "typing.Generic": pyval_template = pyval.parameters else: pyval_template = base.template template = tuple(t.name for t in pyval_template) parameters = pyval.parameters assert (pyval.name in ("typing.Generic", "typing.Protocol") or len(parameters) <= len(template)) # Delay type parameter loading to handle recursive types. # See the ParameterizedClass.formal_type_parameters() property. type_parameters = abstract_utils.LazyFormalTypeParameters( template, parameters, subst) return abstract_class(base_cls, type_parameters, self.ctx) elif isinstance(pyval, pytd.Literal): value = self.constant_to_value( self._get_literal_value(pyval.value), subst, self.ctx.root_node) return abstract.LiteralClass(value, self.ctx) elif isinstance(pyval, pytd.Annotated): typ = self.constant_to_value(pyval.base_type, subst, self.ctx.root_node) if pyval.annotations[0] == "'pytype_metadata'": try: md = metadata.from_string(pyval.annotations[1]) if md["tag"] == "attr.ib": ret = attr_overlay.AttribInstance.from_metadata( self.ctx, self.ctx.root_node, typ, md) return ret elif md["tag"] == "attr.s": ret = attr_overlay.Attrs.from_metadata(self.ctx, md) return ret except (IndexError, ValueError, TypeError, KeyError): details = "Wrong format for pytype_metadata." self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames, pyval.annotations[1], details) return typ else: return typ elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.ctx.root_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))