def test_union_type_ne(self): u1 = pytd.UnionType((self.int, self.float)) u2 = pytd.UnionType((self.float, self.int, self.none_type)) self.assertNotEqual(u1, u2) self.assertNotEqual(u2, u1) self.assertEqual(u1.type_list, (self.int, self.float)) self.assertEqual(u2.type_list, (self.float, self.int, self.none_type))
def testUnionTypeEq(self): u1 = pytd.UnionType((self.int, self.float)) u2 = pytd.UnionType((self.float, self.int)) self.assertEqual(u1, u2) self.assertEqual(u2, u1) self.assertEqual(u1.type_list, (self.int, self.float)) self.assertEqual(u2.type_list, (self.float, self.int))
def VisitGenericType(self, t): module, name = self._GetModuleAndName(t) if self._IsTyping(module): if name == "Optional": return pytd.UnionType(t.parameters + (pytd.NamedType("NoneType"),)) elif name == "Union": return pytd.UnionType(t.parameters) return t
def testOrder(self): # pytd types' primary sort key is the class name, second sort key is # the contents when interpreted as a (named)tuple. nodes = [pytd.AnythingType(), pytd.GenericType(self.list, (self.int,)), pytd.NamedType("int"), pytd.NothingType(), pytd.UnionType((self.float,)), pytd.UnionType((self.int,))] for n1, n2 in zip(nodes[:-1], nodes[1:]): self.assertLess(n1, n2) self.assertLessEqual(n1, n2) self.assertGreater(n2, n1) self.assertGreaterEqual(n2, n1) for p in itertools.permutations(nodes): self.assertEquals(list(sorted(p)), nodes)
def JoinTypes(types): """Combine a list of types into a union type, if needed. Leaves singular return values alone, or wraps a UnionType around them if there are multiple ones, or if there are no elements in the list (or only NothingType) return NothingType. Arguments: types: A list of types. This list might contain other UnionTypes. If so, they are flattened. Returns: A type that represents the union of the types passed in. Order is preserved. """ queue = collections.deque(types) seen = set() new_types = [] while queue: t = queue.popleft() if isinstance(t, pytd.UnionType): queue.extendleft(reversed(t.type_list)) elif isinstance(t, pytd.NothingType): pass elif t not in seen: new_types.append(t) seen.add(t) if len(new_types) == 1: return new_types.pop() elif any(isinstance(t, pytd.AnythingType) for t in new_types): return pytd.AnythingType() elif new_types: return pytd.UnionType(tuple(new_types)) # tuple() to make unions hashable else: return pytd.NothingType()
def _get_parameters(self, t1, t2): if isinstance(t1, pytd.TupleType) and isinstance(t2, pytd.TupleType): # No change needed; the parameters will be compared element-wise. return t1.parameters, t2.parameters elif isinstance(t2, pytd.TupleType): # Since we call _get_parameters after confirming that t1 and t2 have # compatible base types, t1 is a homogeneous tuple here. return (t1.element_type,) * len(t2.parameters), t2.parameters elif isinstance(t1, pytd.TupleType): return (pytd.UnionType(type_list=t1.parameters),), t2.parameters elif (isinstance(t1, pytd.CallableType) and isinstance(t2, pytd.CallableType)): # Flip the arguments, since argument types are contravariant. return t2.args + (t1.ret,), t1.args + (t2.ret,) elif (t1.base_type.cls.name == "__builtin__.type" and t2.base_type.cls.name == "typing.Callable"): # We'll only check the return type, since getting the argument types for # initializing a class is tricky. return t1.parameters, (t2.parameters[-1],) elif (t1.base_type.cls.name == "typing.Callable" and t2.base_type.cls.name == "__builtin__.type"): return (t1.parameters[-1],), t2.parameters elif isinstance(t1, pytd.CallableType): # We're matching against GenericType(Callable, (Any, _RET)), so we don't # need the argument types. return (pytd.AnythingType(), t1.ret), t2.parameters elif isinstance(t2, pytd.CallableType): return t1.parameters, (pytd.AnythingType(), t2.ret) else: num_extra_params = len(t1.parameters) - len(t2.parameters) # Matching, e.g., Dict[str, int] against Iterable[K] is legitimate. assert num_extra_params >= 0, (t1.base_type.cls.name, t2.base_type.cls.name) t2_parameters = t2.parameters + (pytd.AnythingType(),) * num_extra_params return t1.parameters, t2_parameters
def test_join_optional_anything_types(self): """Test that JoinTypes() simplifies unions containing 'Any' and 'None'.""" any_type = pytd.AnythingType() none_type = pytd.NamedType("builtins.NoneType") types = [pytd.NamedType("a"), any_type, none_type] self.assertEqual(pytd_utils.JoinTypes(types), pytd.UnionType((any_type, none_type)))
def match_Generic_against_Generic(self, t1, t2, subst): # pylint: disable=invalid-name """Match a pytd.GenericType against another pytd.GenericType.""" assert isinstance(t1.base_type, pytd.ClassType), type(t1.base_type) assert isinstance(t2.base_type, pytd.ClassType), type(t2.base_type) base1 = pytd.ClassType(t1.base_type.cls.name, t1.base_type.cls) base2 = pytd.ClassType(t2.base_type.cls.name, t2.base_type.cls) base_type_cmp = self.match_type_against_type(base1, base2, subst) if base_type_cmp is booleq.FALSE: return booleq.FALSE if not isinstance(t1, pytd.TupleType) and isinstance( t2, pytd.TupleType): p1, = t1.parameters param_cmp = [ self.match_type_against_type(p1, p2, subst) for p2 in t2.parameters ] else: t1_parameters = t1.parameters if isinstance(t1, pytd.TupleType): if isinstance(t2, pytd.TupleType): if len(t1_parameters) != len(t2.parameters): return booleq.FALSE else: t1_parameters = (pytd.UnionType(type_list=t1_parameters), ) # Matching, e.g., Dict[str, int] against Iterable[K] is legitimate. assert len(t1_parameters) >= len( t2.parameters), t1.base_type.cls.name # Type parameters are covariant: # E.g. passing list[int] as argument for list[object] succeeds. param_cmp = [ self.match_type_against_type(p1, p2, subst) for p1, p2 in zip(t1_parameters, t2.parameters) ] return booleq.And([base_type_cmp] + param_cmp)
def testComplexCombinedType(self): """Test parsing a type with both union and intersection.""" data1 = r"def foo(a: Foo or Bar and Zot) -> object: ..." data2 = r"def foo(a: Foo or (Bar and Zot)) -> object: ..." result1 = self.Parse(data1) result2 = self.Parse(data2) f = pytd.Function( name="foo", signatures=(pytd.Signature(params=(pytd.Parameter( name="a", type=pytd.UnionType( type_list=(pytd.NamedType("Foo"), pytd.IntersectionType( type_list=(pytd.NamedType("Bar"), pytd.NamedType("Zot"))))), kwonly=False, optional=False, mutated_type=None), ), starargs=None, starstarargs=None, return_type=pytd.NamedType("object"), template=(), exceptions=()), ), kind=pytd.METHOD) self.assertEqual(f, result1.Lookup("foo")) self.assertEqual(f, result2.Lookup("foo"))
def value_to_pytd_type(self, node, v, seen, view): """Get a PyTD type representing this object, as seen at a node. Args: node: The node from which we want to observe this object. v: The object. seen: The set of values seen before while computing the type. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, (abstract.Empty, abstract.Nothing)): return pytd.NothingType() elif isinstance(v, abstract.TypeParameterInstance): if (v.name in v.instance.type_parameters and v.instance.type_parameters[v.name].bindings): return pytd_utils.JoinTypes( self.value_to_pytd_type(node, p, seen, view) for p in v.instance.type_parameters[v.name].data) else: # The type parameter was never initialized return pytd.AnythingType() elif isinstance(v, (abstract.Function, abstract.IsInstance, abstract.BoundFunction)): return pytd.NamedType("typing.Callable") elif isinstance(v, abstract.Class): param = self.value_instance_to_pytd_type(node, v, None, seen, view) return pytd.GenericType(base_type=pytd.NamedType("__builtin__.type"), parameters=(param,)) elif isinstance(v, abstract.Module): return pytd.NamedType("__builtin__.module") elif isinstance(v, abstract.SimpleAbstractValue): if v.cls: classvalues = self._get_values(node, v.cls, view) cls_types = [] for cls in classvalues: cls_types.append(self.value_instance_to_pytd_type( node, cls, v, seen=seen, view=view)) ret = pytd_utils.JoinTypes(cls_types) visitors.InPlaceFillInClasses(ret, v.vm.loader.builtins) return ret else: # We don't know this type's __class__, so return AnythingType to # indicate that we don't know anything about what this is. # This happens e.g. for locals / globals, which are returned from the # code in class declarations. log.info("Using ? for %s", v.name) return pytd.AnythingType() elif isinstance(v, abstract.Union): return pytd.UnionType(tuple(self.value_to_pytd_type(node, o, seen, view) for o in v.options)) elif isinstance(v, abstract.SuperInstance): return pytd.NamedType("__builtin__.super") elif isinstance(v, abstract.Unsolvable): return pytd.AnythingType() elif isinstance(v, abstract.Unknown): return pytd.NamedType(v.class_name) else: raise NotImplementedError(v.__class__.__name__)
def value_instance_to_pytd_type(self, node, v, instance, seen, view): """Get the PyTD type an instance of this object would have. Args: node: The node. v: The object. instance: The instance. seen: Already seen instances. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, abstract.Union): return pytd.UnionType(tuple( self.value_instance_to_pytd_type(node, t, instance, seen, view) for t in v.options)) elif isinstance(v, abstract.AnnotationContainer): return self.value_instance_to_pytd_type( node, v.base_cls, instance, seen, view) elif isinstance(v, mixin.Class): if not self._detailed and v.official_name is None: return pytd.AnythingType() if seen is None: # We make the set immutable to ensure that the seen instances for # different parameter values don't interfere with one another. seen = frozenset() if instance in seen: # We have a circular dependency in our types (e.g., lst[0] == lst). Stop # descending into the type parameters. type_params = () else: type_params = tuple(t.name for t in v.template) if instance is not None: seen |= {instance} type_arguments = self._value_to_parameter_types( node, v, instance, type_params, seen, view) base = pytd_utils.NamedTypeWithModule(v.official_name or v.name, v.module) if self._is_tuple(v, instance): if type_arguments: homogeneous = False else: homogeneous = True type_arguments = [pytd.NothingType()] elif v.full_name == "typing.Callable": homogeneous = not isinstance(v, abstract.CallableClass) else: homogeneous = len(type_arguments) == 1 return pytd_utils.MakeClassOrContainerType( base, type_arguments, homogeneous) elif isinstance(v, abstract.TypeParameter): # We generate the full definition because, if this type parameter is # imported, we will need the definition in order to declare it later. return self._typeparam_to_def(node, v, v.name) elif isinstance(v, typing_overlay.NoReturn): return pytd.NothingType() else: log.info("Using ? for instance of %s", v.name) return pytd.AnythingType()
def p_type_homogeneous(self, p): """type : named_or_external_type LBRACKET parameters RBRACKET""" _, base_type, _, parameters, _ = p if p[1] == pytd.NamedType('Union'): p[0] = pytd.UnionType(parameters) elif p[1] == pytd.NamedType('Optional'): p[0] = pytd.UnionType(parameters[0], pytd.NamedType('None')) elif len(parameters) == 2 and parameters[-1] is Ellipsis: element_type, _ = parameters if element_type is Ellipsis: make_syntax_error(self, '[..., ...] not supported', p) p[0] = pytd.HomogeneousContainerType(base_type=base_type, parameters=(element_type, )) else: parameters = tuple(pytd.AnythingType() if p is Ellipsis else p for p in parameters) p[0] = pytd.GenericType(base_type=base_type, parameters=parameters)
def test_metaclass_union(self): cls = abstract.InterpreterClass("X", [], {}, None, self._vm) meta1 = abstract.InterpreterClass("M1", [], {}, None, self._vm) meta2 = abstract.InterpreterClass("M2", [], {}, None, self._vm) meta1.official_name = "M1" meta2.official_name = "M2" cls.cls = abstract.Union([meta1, meta2], self._vm) pytd_cls = cls.to_pytd_def(self._vm.root_cfg_node, "X") self.assertEqual(pytd_cls.metaclass, pytd.UnionType( (pytd.NamedType("M1"), pytd.NamedType("M2"))))
def value_instance_to_pytd_type(self, node, v, instance, seen, view): """Get the PyTD type an instance of this object would have. Args: node: The node. v: The object. instance: The instance. seen: Already seen instances. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, abstract.Union): return pytd.UnionType(tuple( self.value_instance_to_pytd_type(node, t, instance, seen, view) for t in v.options)) elif isinstance(v, abstract.AnnotationContainer): return self.value_instance_to_pytd_type( node, v.base_cls, instance, seen, view) elif isinstance(v, abstract.Class): if v.official_name is None: return pytd.AnythingType() if seen is None: seen = set() if instance in seen: # We have a circular dependency in our types (e.g., lst[0] == lst). Stop # descending into the type parameters. type_params = () else: type_params = tuple(t.name for t in v.template) if instance is not None: seen.add(instance) type_arguments = self._value_to_parameter_types( node, v, instance, type_params, seen, view) base = pytd_utils.NamedTypeWithModule(v.official_name, v.module) if self._is_tuple(v, instance): if type_arguments: homogeneous = False else: homogeneous = True type_arguments = [pytd.NothingType()] else: homogeneous = len(type_arguments) == 1 return pytd_utils.MakeClassOrContainerType( base, type_arguments, homogeneous) elif isinstance(v, abstract.TypeVariable): return pytd.TypeParameter(v.name, None) else: log.info("Using ? for instance of %s", v.name) return pytd.AnythingType()
def setUp(self): self.options = config.Options.create( python_version=self.PYTHON_VERSION, python_exe=self.PYTHON_EXE) def t(name): # pylint: disable=invalid-name return pytd.ClassType("__builtin__." + name) self.bool = t("bool") self.dict = t("dict") self.float = t("float") self.complex = t("complex") self.int = t("int") self.list = t("list") self.none_type = t("NoneType") self.object = t("object") self.set = t("set") self.frozenset = t("frozenset") self.str = t("str") self.bytearray = t("bytearray") self.tuple = t("tuple") self.unicode = t("unicode") self.generator = t("generator") self.function = pytd.ClassType("typing.Callable") self.anything = pytd.AnythingType() self.nothing = pytd.NothingType() self.module = t("module") self.file = t("file") # The various union types use pytd_utils.CanonicalOrdering()'s ordering: self.intorstr = pytd.UnionType((self.int, self.str)) self.strorunicode = pytd.UnionType((self.str, self.unicode)) self.intorfloat = pytd.UnionType((self.float, self.int)) self.intorfloatorstr = pytd.UnionType((self.float, self.int, self.str)) self.complexorstr = pytd.UnionType((self.complex, self.str)) self.intorfloatorcomplex = pytd.UnionType( (self.int, self.float, self.complex)) self.int_tuple = pytd.GenericType(self.tuple, (self.int, )) self.nothing_tuple = pytd.GenericType(self.tuple, (self.nothing, )) self.intorfloat_tuple = pytd.GenericType(self.tuple, (self.intorfloat, )) self.int_set = pytd.GenericType(self.set, (self.int, )) self.intorfloat_set = pytd.GenericType(self.set, (self.intorfloat, )) self.unknown_frozenset = pytd.GenericType(self.frozenset, (self.anything, )) self.float_frozenset = pytd.GenericType(self.frozenset, (self.float, )) self.empty_frozenset = pytd.GenericType(self.frozenset, (self.nothing, )) self.int_list = pytd.GenericType(self.list, (self.int, )) self.str_list = pytd.GenericType(self.list, (self.str, )) self.intorfloat_list = pytd.GenericType(self.list, (self.intorfloat, )) self.intorstr_list = pytd.GenericType(self.list, (self.intorstr, )) self.anything_list = pytd.GenericType(self.list, (self.anything, )) self.nothing_list = pytd.GenericType(self.list, (self.nothing, )) self.int_int_dict = pytd.GenericType(self.dict, (self.int, self.int)) self.int_str_dict = pytd.GenericType(self.dict, (self.int, self.str)) self.str_int_dict = pytd.GenericType(self.dict, (self.str, self.int)) self.nothing_nothing_dict = pytd.GenericType( self.dict, (self.nothing, self.nothing))
def setUpClass(cls): super().setUpClass() # We use class-wide loader to avoid creating a new loader for every test # method if not required. cls._loader = None def t(name): # pylint: disable=invalid-name return pytd.ClassType("builtins." + name) cls.bool = t("bool") cls.dict = t("dict") cls.float = t("float") cls.complex = t("complex") cls.int = t("int") cls.list = t("list") cls.none_type = t("NoneType") cls.object = t("object") cls.set = t("set") cls.frozenset = t("frozenset") cls.str = t("str") cls.bytearray = t("bytearray") cls.tuple = t("tuple") cls.unicode = t("unicode") cls.generator = t("generator") cls.function = pytd.ClassType("typing.Callable") cls.anything = pytd.AnythingType() cls.nothing = pytd.NothingType() cls.module = t("module") cls.file = t("file") # The various union types use pytd_utils.CanonicalOrdering()'s ordering: cls.intorstr = pytd.UnionType((cls.int, cls.str)) cls.strorunicode = pytd.UnionType((cls.str, cls.unicode)) cls.intorfloat = pytd.UnionType((cls.float, cls.int)) cls.intorfloatorstr = pytd.UnionType((cls.float, cls.int, cls.str)) cls.complexorstr = pytd.UnionType((cls.complex, cls.str)) cls.intorfloatorcomplex = pytd.UnionType( (cls.int, cls.float, cls.complex)) cls.int_tuple = pytd.GenericType(cls.tuple, (cls.int, )) cls.nothing_tuple = pytd.TupleType(cls.tuple, ()) cls.intorfloat_tuple = pytd.GenericType(cls.tuple, (cls.intorfloat, )) cls.int_set = pytd.GenericType(cls.set, (cls.int, )) cls.intorfloat_set = pytd.GenericType(cls.set, (cls.intorfloat, )) cls.unknown_frozenset = pytd.GenericType(cls.frozenset, (cls.anything, )) cls.float_frozenset = pytd.GenericType(cls.frozenset, (cls.float, )) cls.empty_frozenset = pytd.GenericType(cls.frozenset, (cls.nothing, )) cls.int_list = pytd.GenericType(cls.list, (cls.int, )) cls.str_list = pytd.GenericType(cls.list, (cls.str, )) cls.intorfloat_list = pytd.GenericType(cls.list, (cls.intorfloat, )) cls.intorstr_list = pytd.GenericType(cls.list, (cls.intorstr, )) cls.anything_list = pytd.GenericType(cls.list, (cls.anything, )) cls.nothing_list = pytd.GenericType(cls.list, (cls.nothing, )) cls.int_int_dict = pytd.GenericType(cls.dict, (cls.int, cls.int)) cls.int_str_dict = pytd.GenericType(cls.dict, (cls.int, cls.str)) cls.str_int_dict = pytd.GenericType(cls.dict, (cls.str, cls.int)) cls.nothing_nothing_dict = pytd.GenericType(cls.dict, (cls.nothing, cls.nothing))
def _normal_param(name, param_type, default, kwonly): """Return a pytd.Parameter object for a normal argument.""" if default is not None: default_type = _type_for_default(default) if param_type is None: param_type = default_type elif default_type == pytd.NamedType("NoneType"): param_type = pytd.UnionType((param_type, default_type)) if param_type is None: # TODO(kramm): We should use __builtin__.object. (And other places) param_type = pytd.NamedType("object") optional = default is not None return pytd.Parameter(name, param_type, kwonly, optional, None)
def _normal_param(name, param_type, default, kwonly): """Return a pytd.Parameter object for a normal argument.""" if default is not None: default_type = _type_for_default(default) if default_type == pytd.NamedType("NoneType"): if param_type is not None: param_type = pytd.UnionType((param_type, default_type)) elif param_type is None: param_type = default_type if param_type is None: param_type = pytd.AnythingType() optional = default is not None return pytd.Parameter(name, param_type, kwonly, optional, None)
def testListConcatMultiType(self): ty = self.Infer(""" def f(): x = [] x.append(1) x.append("str") return x + [1.3] + x f() """, deep=False, show_library_calls=True) self.assertHasOnlySignatures( ty.Lookup("f"), ((), pytd.GenericType(self.list, (pytd.UnionType( (self.int, self.float, self.str)), ))))
def testJoinTypes(self): """Test that JoinTypes() does recursive flattening.""" n1, n2, n3, n4, n5, n6 = [pytd.NamedType("n%d" % i) for i in xrange(6)] # n1 or (n2 or (n3)) nested1 = pytd.UnionType((n1, pytd.UnionType((n2, pytd.UnionType((n3,)))))) # ((n4) or n5) or n6 nested2 = pytd.UnionType((pytd.UnionType((pytd.UnionType((n4,)), n5)), n6)) joined = pytd_utils.JoinTypes([nested1, nested2]) self.assertEqual(joined.type_list, (n1, n2, n3, n4, n5, n6))
def match_Unknown_against_Generic(self, t1, t2, subst): # pylint: disable=invalid-name assert isinstance(t2.base_type, pytd.ClassType) # No inheritance for base classes - you can only inherit from an # instantiated template, but not from a template itself. base_match = booleq.Eq(t1.name, t2.base_type.cls.name) type_params = [self.type_parameter(t1, t2.base_type.cls, item) for item in t2.base_type.cls.template] for type_param in type_params: self.solver.register_variable(type_param.name) if isinstance(t2, pytd.TupleType): t2_parameters = (pytd.UnionType(type_list=t2.parameters),) else: t2_parameters = t2.parameters params = [self.match_type_against_type(p1, p2, subst) for p1, p2 in zip(type_params, t2_parameters)] return booleq.And([base_match] + params)
def match_Function_against_Class(self, f1, cls2, subst, cache): cls2_methods = cache.get(id(cls2)) if cls2_methods is None: cls2_methods = cache[id(cls2)] = {f.name: f for f in cls2.methods} if f1.name not in cls2_methods: # The class itself doesn't have this method, but base classes might. # TODO(kramm): This should do MRO order, not depth-first. for base in cls2.parents: if isinstance(base, pytd.AnythingType): # AnythingType can contain any method. However, that would mean that # a class that inherits from AnythingType contains any method # imaginable, and hence is a match for anything. To prevent the bad # results caused by that, return FALSE here. return booleq.FALSE elif isinstance(base, (pytd.ClassType, pytd.GenericType)): if isinstance(base, pytd.ClassType): cls = base.cls values = tuple(pytd.AnythingType() for _ in cls.template) elif isinstance(base, pytd.TupleType): cls = base.base_type.cls values = (pytd.UnionType(type_list=base.parameters), ) else: cls = base.base_type.cls values = base.parameters if values: subst = subst.copy() for param, value in zip(cls.template, values): subst[param.type_param] = value implication = self.match_Function_against_Class( f1, cls, subst, cache) if implication is not booleq.FALSE: return implication else: # Funky types like UnionType are hard to match against (and shouldn't # appear as a base class) so we treat them as catch-all. log.warning("Assuming that %s has method %s", pytd_utils.Print(base), f1.name) return booleq.TRUE return booleq.FALSE else: f2 = cls2_methods[f1.name] return self.match_Function_against_Function(f1, f2, subst, skip_self=True)
def testListConcatMultiType(self): with self.Infer(""" def f(): x = [] x.append(1) x.append("str") return x + [1.3] + x f() """, deep=False, solve_unknowns=False, extract_locals=False) as ty: self.assertHasOnlySignatures( ty.Lookup("f"), ((), pytd.HomogeneousContainerType(self.list, (pytd.UnionType( (self.int, self.float, self.str)), ))))
def VisitUnionType(self, union): """Push unions down into containers. This collects similar container types in unions and merges them into single instances with the union type pushed down to the element_type level. Arguments: union: A pytd.Union instance. Might appear in a parameter, a return type, a constant type, etc. Returns: A simplified pytd.Union. """ if not any(isinstance(t, pytd.GenericType) for t in union.type_list): # Optimization: If we're not going to change anything, return original. return union union = utils.JoinTypes(union.type_list) # flatten if not isinstance(union, pytd.UnionType): union = pytd.UnionType((union, )) collect = {} has_redundant_base_types = False for t in union.type_list: if isinstance(t, pytd.GenericType): if t.base_type in collect: has_redundant_base_types = True collect[t.base_type] = tuple( utils.JoinTypes([p1, p2]) for p1, p2 in zip(collect[t.base_type], t.parameters)) else: collect[t.base_type] = t.parameters if not has_redundant_base_types: return union result = pytd.NothingType() done = set() for t in union.type_list: if isinstance(t, pytd.GenericType): if t.base_type in done: continue # already added parameters = collect[t.base_type] add = t.Replace(parameters=tuple( p.Visit(CombineContainers()) for p in parameters)) done.add(t.base_type) else: add = t result = utils.JoinTypes([result, add]) return result
def p_type_or(self, p): """type : type OR type""" # This rule depends on precedence specification # UnionType flattens any contained UnionType's p[0] = pytd.UnionType((p[1], p[3]))
def _constant_to_value(self, pyval, subst, get_node): """Create a AtomicAbstractValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if pyval.__class__ is str: # We use a subclass of str, compat.BytesPy3, to mark Python 3 # bytestrings, which are converted to abstract bytes instances. # compat.BytesType dispatches to this when appropriate. return abstract.AbstractOrConcreteValue(pyval, self.str_type, self.vm) elif isinstance(pyval, compat.UnicodeType): return abstract.AbstractOrConcreteValue(pyval, self.unicode_type, self.vm) elif isinstance(pyval, compat.BytesType): return abstract.AbstractOrConcreteValue(pyval, self.bytes_type, self.vm) elif isinstance(pyval, bool): return self.true if pyval is True else self.false elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.AbstractOrConcreteValue(pyval, self.int_type, self.vm) elif isinstance(pyval, compat.LongType): # long is aliased to int return self.primitive_class_instances[int] elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.AbstractOrConcreteValue( pyval, self.primitive_classes[types.CodeType], self.vm) elif pyval is super: return special_builtins.Super(self.vm) elif pyval is object: return special_builtins.Object(self.vm) elif pyval.__class__ is type: try: return self.name_to_value(self._type_to_name(pyval), subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.LateType): actual = self._load_late_type(pyval) return self._constant_to_value(actual, subst, get_node) elif isinstance(pyval, pytd.TypeDeclUnit): return self._create_module(pyval) elif isinstance(pyval, pytd.Module): mod = self.vm.loader.import_name(pyval.module_name) return self._create_module(mod) elif isinstance(pyval, pytd.Class): if pyval.name == "__builtin__.super": return self.vm.special_builtins["super"] elif pyval.name == "__builtin__.object": return self.object_type elif pyval.name == "types.ModuleType": return self.module_type elif pyval.name == "_importlib_modulespec.ModuleType": # Python 3's typeshed uses a stub file indirection to define ModuleType # even though it is exported via types.pyi. return self.module_type else: module, dot, base_name = pyval.name.rpartition(".") try: cls = abstract.PyTDClass(base_name, pyval, self.vm) except mro.MROError as e: self.vm.errorlog.mro_error(self.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module return cls elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.vm) for sig in pyval.signatures ] type_new = self.vm.lookup_builtin("__builtin__.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.vm) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): assert pyval.cls return self.constant_to_value(pyval.cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.NothingType): return self.empty elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.AnythingType)): # We allow "X = ... # type: Any" to declare X as a type. return self.unsolvable elif isinstance(pyval, pytd.FunctionType): return self.constant_to_value(pyval.function, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.vm.root_cfg_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.vm) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.vm.root_cfg_node) for c in pyval.constraints) bound = (pyval.bound and self.constant_to_value( pyval.bound, {}, self.vm.root_cfg_node)) return abstract.TypeParameter(pyval.name, self.vm, constraints=constraints, bound=bound) elif isinstance(pyval, abstract.AsInstance): cls = pyval.cls if isinstance(cls, pytd.LateType): actual = self._load_late_type(cls) if not isinstance(actual, pytd.ClassType): return self.unsolvable cls = actual.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if (isinstance(cls, pytd.GenericType) and cls.base_type.name == "typing.ClassVar"): param, = cls.parameters return self.constant_to_value(abstract.AsInstance(param), subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and cls.template): # If we're converting a generic Class, need to create a new instance of # it. See test_classes.testGenericReinstantiated. if isinstance(cls, pytd.Class): params = tuple(t.type_param.upper_value for t in cls.template) cls = pytd.GenericType(base_type=pytd.ClassType( cls.name, cls), parameters=params) if isinstance(cls.base_type, pytd.LateType): actual = self._load_late_type(cls.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base_cls = actual.cls else: assert isinstance(cls.base_type, pytd.ClassType) base_cls = cls.base_type.cls assert isinstance(base_cls, pytd.Class), base_cls if base_cls.name == "__builtin__.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.name not in subst: raise self.TypeParameterError(c.name) return self.merge_classes(get_node(), subst[c.name].data) else: return self.constant_to_value(c, subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.TupleType): content = tuple( self.constant_to_var(abstract.AsInstance(p), subst, get_node()) for p in cls.parameters) return abstract.Tuple(content, self.vm) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.vm.root_cfg_node) return abstract.Instance(clsval, self.vm) else: clsval = self.constant_to_value(base_cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(clsval, self.vm) num_params = len(cls.parameters) assert num_params <= len(base_cls.template) for i, formal in enumerate(base_cls.template): if i < num_params: node = get_node() p = self.constant_to_var( abstract.AsInstance(cls.parameters[i]), subst, node) else: # An omitted type parameter implies `Any`. node = self.vm.root_cfg_node p = self.unsolvable.to_variable(node) instance.merge_type_parameter(node, formal.name, p) return instance elif isinstance(cls, pytd.Class): assert not cls.template # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in [ "__builtin__.type", "__builtin__.property" ]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(mycls, self.vm) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] else: return self.constant_to_value(cls, subst, self.vm.root_cfg_node) elif (isinstance(pyval, pytd.GenericType) and pyval.base_type.name == "typing.ClassVar"): param, = pyval.parameters return self.constant_to_value(param, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.GenericType): if isinstance(pyval.base_type, pytd.LateType): actual = self._load_late_type(pyval.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base = actual.cls else: assert isinstance(pyval.base_type, pytd.ClassType) base = pyval.base_type.cls assert isinstance(base, pytd.Class), base base_cls = self.constant_to_value(base, subst, self.vm.root_cfg_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = list(range(len(pyval.parameters))) + [abstract.T] parameters = pyval.parameters + (pytd.UnionType( pyval.parameters), ) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.Callable template = list(range(len( pyval.args))) + [abstract.ARGS, abstract.RET] parameters = pyval.args + (pytd_utils.JoinTypes( pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass template = tuple(t.name for t in base.template) parameters = pyval.parameters assert (pyval.base_type.name == "typing.Generic" or len(parameters) <= len(template)) type_parameters = datatypes.LazyDict() for i, name in enumerate(template): if i < len(parameters): type_parameters.add_lazy_item(name, self.constant_to_value, parameters[i], subst, self.vm.root_cfg_node) else: type_parameters[name] = self.unsolvable return abstract_class(base_cls, type_parameters, self.vm) elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.vm.root_cfg_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def value_to_pytd_type(self, node, v, seen, view): """Get a PyTD type representing this object, as seen at a node. Args: node: The node from which we want to observe this object. v: The object. seen: The set of values seen before while computing the type. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)): return pytd.NothingType() elif isinstance(v, abstract.TypeParameterInstance): if v.instance.get_instance_type_parameter(v.full_name).bindings: # The type parameter was initialized. Set the view to None, since we # don't include v.instance in the view. return pytd_utils.JoinTypes( self.value_to_pytd_type(node, p, seen, None) for p in v.instance.get_instance_type_parameter(v.full_name).data) elif v.param.constraints: return pytd_utils.JoinTypes( self.value_instance_to_pytd_type(node, p, None, seen, view) for p in v.param.constraints) elif v.param.bound: return self.value_instance_to_pytd_type( node, v.param.bound, None, seen, view) else: return pytd.AnythingType() elif isinstance(v, typing_overlay.TypeVar): return pytd.NamedType("__builtin__.type") elif isinstance(v, abstract.FUNCTION_TYPES): try: signatures = abstract_utils.get_signatures(v) except NotImplementedError: return pytd.NamedType("typing.Callable") if len(signatures) == 1: val = self.signature_to_callable(signatures[0], self.vm) if (not isinstance(v, abstract.PYTD_FUNCTION_TYPES) or not self.vm.annotations_util.get_type_parameters(val)): # This is a workaround to make sure we don't put unexpected type # parameters in call traces. return self.value_instance_to_pytd_type(node, val, None, seen, view) return pytd.NamedType("typing.Callable") elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)): return self.value_to_pytd_type(node, v.method, seen, view) elif isinstance(v, (special_builtins.IsInstance, special_builtins.ClassMethodCallable)): return pytd.NamedType("typing.Callable") elif isinstance(v, mixin.Class): param = self.value_instance_to_pytd_type(node, v, None, seen, view) return pytd.GenericType(base_type=pytd.NamedType("__builtin__.type"), parameters=(param,)) elif isinstance(v, abstract.Module): return pytd.NamedType("__builtin__.module") elif isinstance(v, abstract.SimpleAbstractValue): if v.cls: ret = self.value_instance_to_pytd_type( node, v.cls, v, seen=seen, view=view) ret.Visit(visitors.FillInLocalPointers( {"__builtin__": self.vm.loader.builtins})) return ret else: # We don't know this type's __class__, so return AnythingType to # indicate that we don't know anything about what this is. # This happens e.g. for locals / globals, which are returned from the # code in class declarations. log.info("Using ? for %s", v.name) return pytd.AnythingType() elif isinstance(v, abstract.Union): return pytd.UnionType(tuple(self.value_to_pytd_type(node, o, seen, view) for o in v.options)) elif isinstance(v, special_builtins.SuperInstance): return pytd.NamedType("__builtin__.super") elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)): # Arguably, the type of a type parameter is NamedType("typing.TypeVar"), # but pytype doesn't know how to handle that, so let's just go with Any. return pytd.AnythingType() elif isinstance(v, abstract.Unknown): return pytd.NamedType(v.class_name) elif isinstance(v, abstract.BuildClass): return pytd.NamedType("typing.Callable") else: raise NotImplementedError(v.__class__.__name__)
def VisitUnionType(self, node): return pytd.UnionType(tuple(sorted(node.type_list)))
def new_union_type(self, types): """Return a new UnionType composed of the specified types.""" # UnionType flattens any contained UnionType's. return pytd.UnionType(tuple(types))
def testAnyReplacement(self): union = pytd.UnionType((pytd.NamedType("a"), pytd.NamedType("b"))) self.assertEqual( union.Visit(visitors.ReplaceUnionsWithAny()), pytd.AnythingType())