def test_callable_with_args(self): ast = self._load_ast("a", """ from typing import Callable x = ... # type: Callable[[int, bool], str] """) x = ast.Lookup("a.x").type cls = self._vm.convert.constant_to_value(x, {}, self._vm.root_cfg_node) instance = self._vm.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._vm.root_cfg_node) self.assertIsInstance(cls, abstract.CallableClass) six.assertCountEqual( self, cls.formal_type_parameters.items(), [(0, self._vm.convert.int_type), (1, self._vm.convert.primitive_classes[bool]), (abstract_utils.ARGS, abstract.Union( [cls.formal_type_parameters[0], cls.formal_type_parameters[1]], self._vm)), (abstract_utils.RET, self._vm.convert.str_type)]) self.assertIsInstance(instance, abstract.Instance) self.assertEqual(instance.cls, cls) six.assertCountEqual( self, [(name, set(var.data)) for name, var in instance.instance_type_parameters.items()], [(abstract_utils.full_type_name(instance, abstract_utils.ARGS), {self._vm.convert.primitive_class_instances[int], self._vm.convert.primitive_class_instances[bool]}), (abstract_utils.full_type_name(instance, abstract_utils.RET), {self._vm.convert.primitive_class_instances[str]})])
def test_heterogeneous_tuple(self): ast = self._load_ast("a", """ from typing import Tuple x = ... # type: Tuple[str, int] """) x = ast.Lookup("a.x").type cls = self._vm.convert.constant_to_value(x, {}, self._vm.root_cfg_node) instance = self._vm.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._vm.root_cfg_node) self.assertIsInstance(cls, abstract.TupleClass) six.assertCountEqual(self, cls.formal_type_parameters.items(), [(0, self._vm.convert.str_type), (1, self._vm.convert.int_type), (abstract_utils.T, abstract.Union([ cls.formal_type_parameters[0], cls.formal_type_parameters[1], ], self._vm))]) self.assertIsInstance(instance, abstract.Tuple) self.assertListEqual([v.data for v in instance.pyval], [[self._vm.convert.primitive_class_instances[str]], [self._vm.convert.primitive_class_instances[int]]]) # The order of option elements in Union is random six.assertCountEqual( self, instance.get_instance_type_parameter(abstract_utils.T).data, [self._vm.convert.primitive_class_instances[str], self._vm.convert.primitive_class_instances[int]])
def _get_mutation(self, node, arg_dict, subst): """Mutation for changing the type parameters of mutable arguments. This will adjust the type parameters as needed for pytd functions like: def append_float(x: list[int]): x = list[int or float] This is called after all the signature matching has succeeded, and we know we're actually calling this function. Args: node: The current CFG node. arg_dict: A map of strings to pytd.Bindings instances. subst: Current type parameters. Returns: A list of Mutation instances. Raises: ValueError: If the pytd contains invalid information for mutated params. """ # Handle mutable parameters using the information type parameters mutations = [] # It's possible that the signature contains type parameters that are used # in mutations but are not filled in by the arguments, e.g. when starargs # and starstarargs have type parameters but are not in the args. Check that # subst has an entry for every type parameter, adding any that are missing. if any(f.mutated_type for f in self.pytd_sig.params): subst = subst.copy() for t in pytd_utils.GetTypeParameters(self.pytd_sig): if t.full_name not in subst: subst[t.full_name] = self.vm.convert.empty.to_variable( node) for formal in self.pytd_sig.params: actual = arg_dict[formal.name] arg = actual.data if (formal.mutated_type is not None and arg.isinstance_SimpleValue()): try: all_names_actuals = self._collect_mutated_parameters( formal.type, formal.mutated_type) except ValueError as e: log.error("Old: %s", pytd_utils.Print(formal.type)) log.error("New: %s", pytd_utils.Print(formal.mutated_type)) log.error("Actual: %r", actual) raise ValueError( "Mutable parameters setting a type to a " "different base type is not allowed.") from e for names_actuals in all_names_actuals: for tparam, type_actual in names_actuals: log.info("Mutating %s to %s", tparam.name, pytd_utils.Print(type_actual)) type_actual_val = self.vm.convert.constant_to_var( abstract_utils.AsInstance(type_actual), subst, node, discard_concrete_values=True) mutations.append( Mutation(arg, tparam.full_name, type_actual_val)) return mutations
def test_classvar_instance(self): ast = self._load_ast("a", """ from typing import ClassVar class X: v: ClassVar[int] """) pyval = ast.Lookup("a.X").Lookup("v").type v = self._vm.convert.constant_to_value( abstract_utils.AsInstance(pyval), {}, self._vm.root_cfg_node) self.assertEqual(v, self._vm.convert.primitive_class_instances[int])
def _get_mutation(self, node, arg_dict, subst): """Mutation for changing the type parameters of mutable arguments. This will adjust the type parameters as needed for pytd functions like: def append_float(x: list[int]): x = list[int or float] This is called after all the signature matching has succeeded, and we know we're actually calling this function. Args: node: The current CFG node. arg_dict: A map of strings to pytd.Bindings instances. subst: Current type parameters. Returns: A list of Mutation instances. Raises: ValueError: If the pytd contains invalid information for mutated params. """ # Handle mutable parameters using the information type parameters mutations = [] for formal in self.pytd_sig.params: actual = arg_dict[formal.name] arg = actual.data if (formal.mutated_type is not None and arg.isinstance_SimpleAbstractValue()): if (isinstance(formal.type, pytd.GenericType) and isinstance(formal.mutated_type, pytd.GenericType) and formal.type.base_type == formal.mutated_type.base_type and isinstance(formal.type.base_type, pytd.ClassType) and formal.type.base_type.cls): names_actuals = zip( formal.mutated_type.base_type.cls.template, formal.mutated_type.parameters) for tparam, type_actual in names_actuals: log.info("Mutating %s to %s", tparam.name, pytd_utils.Print(type_actual)) type_actual_val = self.vm.convert.constant_to_var( abstract_utils.AsInstance(type_actual), subst, node, discard_concrete_values=True) mutations.append( Mutation(arg, tparam.full_name, type_actual_val)) else: log.error("Old: %s", pytd_utils.Print(formal.type)) log.error("New: %s", pytd_utils.Print(formal.mutated_type)) log.error("Actual: %r", actual) raise ValueError("Mutable parameters setting a type to a " "different base type is not allowed.") return mutations
def test_callable_no_args(self): ast = self._load_ast("a", """ from typing import Callable x = ... # type: Callable[[], ...] """) x = ast.Lookup("a.x").type cls = self._vm.convert.constant_to_value(x, {}, self._vm.root_cfg_node) instance = self._vm.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._vm.root_cfg_node) self.assertIsInstance( cls.get_formal_type_parameter(abstract_utils.ARGS), abstract.Empty) self.assertEqual(abstract_utils.get_atomic_value( instance.get_instance_type_parameter(abstract_utils.ARGS)), self._vm.convert.empty)
def call(self, node, func, args): if self.vm.PY3: # In Python 3, the type of IO object returned depends on the mode. self.match_args(node, args) # May raise FailedFunctionCall. sig, = self.signatures try: mode = get_file_mode(sig, args) except abstract_utils.ConversionError: pass else: # The default mode is 'r'. io_type = "Binary" if "b" in mode else "Text" return node, self.vm.convert.constant_to_var(abstract_utils.AsInstance( self.vm.lookup_builtin("typing.%sIO" % io_type)), {}, node) return super(Open, self).call(node, func, args)
def call(self, node, func, args): if self.vm.PY3: # In Python 3, the type of IO object returned depends on the mode. self.match_args(node, args) # May raise FailedFunctionCall. sig, = self.signatures callargs = {name: var for name, var, _ in sig.signature.iter_args(args)} try: if "mode" not in callargs: io_type = "Text" # The default mode is 'r'. else: mode = abstract_utils.get_atomic_python_constant(callargs["mode"]) io_type = "Binary" if "b" in mode else "Text" except abstract_utils.ConversionError: pass else: return node, self.vm.convert.constant_to_var(abstract_utils.AsInstance( self.vm.lookup_builtin("typing.%sIO" % io_type)), {}, node) return super(Open, self).call(node, func, args)
def _convert_type(self, t, as_instance=False): """Convenience function for turning a string into an abstract value. Note that this function cannot be called more than once per test with the same arguments, since we hash the arguments to get a filename for the temporary pyi. Args: t: The string representation of a type. as_instance: Whether to convert as an instance. Returns: An AtomicAbstractValue. """ src = "from typing import Any, Callable, Iterator, Tuple, Type\n" src += "from protocols import Sequence, SupportsLower\n" src += "x = ... # type: " + t filename = str(hash((t, as_instance))) x = self._parse_and_lookup(src, "x", filename).type if as_instance: x = abstract_utils.AsInstance(x) return self.vm.convert.constant_to_value(x, {}, self.vm.root_cfg_node)
def test_plain_callable(self): ast = self._load_ast("a", """ from typing import Callable x = ... # type: Callable[..., int] """) x = ast.Lookup("a.x").type cls = self._vm.convert.constant_to_value(x, {}, self._vm.root_cfg_node) instance = self._vm.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._vm.root_cfg_node) self.assertIsInstance(cls, abstract.ParameterizedClass) six.assertCountEqual(self, cls.formal_type_parameters.items(), [(abstract_utils.ARGS, self._vm.convert.unsolvable), (abstract_utils.RET, self._vm.convert.int_type)]) self.assertIsInstance(instance, abstract.Instance) self.assertEqual(instance.cls, cls.base_cls) six.assertCountEqual( self, [(name, var.data) for name, var in instance.instance_type_parameters.items()], [(abstract_utils.full_type_name(instance, abstract_utils.ARGS), [self._vm.convert.unsolvable]), (abstract_utils.full_type_name(instance, abstract_utils.RET), [self._vm.convert.primitive_class_instances[int]])])
def _constant_to_value(self, pyval, subst, get_node): """Create a AtomicAbstractValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if pyval.__class__ is str: # We use a subclass of str, compat.BytesPy3, to mark Python 3 # bytestrings, which are converted to abstract bytes instances. # compat.BytesType dispatches to this when appropriate. return abstract.AbstractOrConcreteValue(pyval, self.str_type, self.vm) elif isinstance(pyval, compat.UnicodeType): return abstract.AbstractOrConcreteValue(pyval, self.unicode_type, self.vm) elif isinstance(pyval, compat.BytesType): return abstract.AbstractOrConcreteValue(pyval, self.bytes_type, self.vm) elif isinstance(pyval, bool): return self.true if pyval else self.false elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.AbstractOrConcreteValue(pyval, self.int_type, self.vm) elif isinstance(pyval, compat.LongType): # long is aliased to int return self.primitive_class_instances[int] elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.AbstractOrConcreteValue( pyval, self.primitive_classes[types.CodeType], self.vm) elif pyval is super: return special_builtins.Super(self.vm) elif pyval is object: return special_builtins.Object(self.vm) elif pyval.__class__ is type: try: return self.name_to_value(self._type_to_name(pyval), subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.LateType): actual = self._load_late_type(pyval) return self._constant_to_value(actual, subst, get_node) elif isinstance(pyval, pytd.TypeDeclUnit): return self._create_module(pyval) elif isinstance(pyval, pytd.Module): mod = self.vm.loader.import_name(pyval.module_name) return self._create_module(mod) elif isinstance(pyval, pytd.Class): if pyval.name == "__builtin__.super": return self.vm.special_builtins["super"] elif pyval.name == "__builtin__.object": return self.object_type elif pyval.name == "types.ModuleType": return self.module_type elif pyval.name == "_importlib_modulespec.ModuleType": # Python 3's typeshed uses a stub file indirection to define ModuleType # even though it is exported via types.pyi. return self.module_type elif pyval.name == "types.FunctionType": return self.function_type else: module, dot, base_name = pyval.name.rpartition(".") # typing.TypingContainer intentionally loads the underlying pytd types. if module != "typing" and module in self.vm.loaded_overlays: overlay = self.vm.loaded_overlays[module] if overlay.get_module(base_name) is overlay: overlay.load_lazy_attribute(base_name) return abstract_utils.get_atomic_value( overlay.members[base_name]) try: cls = abstract.PyTDClass(base_name, pyval, self.vm) except mro.MROError as e: self.vm.errorlog.mro_error(self.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module return cls elif isinstance(pyval, pytd.Function): signatures = [ function.PyTDSignature(pyval.name, sig, self.vm) for sig in pyval.signatures ] type_new = self.vm.lookup_builtin("__builtin__.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.vm) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): assert pyval.cls return self.constant_to_value(pyval.cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.NothingType): return self.empty elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.AnythingType)): # We allow "X = ... # type: Any" to declare X as a type. return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.GenericType) and pyval.type.base_type.name == "__builtin__.type"): # `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`. param, = pyval.type.parameters return self.constant_to_value(param, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.FunctionType): return self.constant_to_value(pyval.function, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.vm.root_cfg_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.vm) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.vm.root_cfg_node) for c in pyval.constraints) bound = (pyval.bound and self.constant_to_value( pyval.bound, {}, self.vm.root_cfg_node)) return abstract.TypeParameter(pyval.name, self.vm, constraints=constraints, bound=bound, module=pyval.scope) elif isinstance(pyval, abstract_utils.AsInstance): cls = pyval.cls if isinstance(cls, pytd.LateType): actual = self._load_late_type(cls) if not isinstance(actual, pytd.ClassType): return self.unsolvable cls = actual.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if (isinstance(cls, pytd.GenericType) and cls.base_type.name == "typing.ClassVar"): param, = cls.parameters return self.constant_to_value(abstract_utils.AsInstance(param), subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and cls.template): # If we're converting a generic Class, need to create a new instance of # it. See test_classes.testGenericReinstantiated. if isinstance(cls, pytd.Class): params = tuple(t.type_param.upper_value for t in cls.template) cls = pytd.GenericType(base_type=pytd.ClassType( cls.name, cls), parameters=params) if isinstance(cls.base_type, pytd.LateType): actual = self._load_late_type(cls.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base_cls = actual.cls else: assert isinstance(cls.base_type, pytd.ClassType) base_cls = cls.base_type.cls assert isinstance(base_cls, pytd.Class), base_cls if base_cls.name == "__builtin__.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.full_name not in subst: raise self.TypeParameterError(c.full_name) return self.merge_classes(subst[c.full_name].data) else: return self.constant_to_value(c, subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.TupleType): content = tuple( self.constant_to_var(abstract_utils.AsInstance( p), subst, get_node()) for p in cls.parameters) return abstract.Tuple(content, self.vm) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.vm.root_cfg_node) return abstract.Instance(clsval, self.vm) else: clsval = self.constant_to_value(base_cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(clsval, self.vm) num_params = len(cls.parameters) assert num_params <= len(base_cls.template) for i, formal in enumerate(base_cls.template): if i < num_params: node = get_node() p = self.constant_to_var( abstract_utils.AsInstance(cls.parameters[i]), subst, node) else: # An omitted type parameter implies `Any`. node = self.vm.root_cfg_node p = self.unsolvable.to_variable(node) instance.merge_instance_type_parameter( node, formal.name, p) return instance elif isinstance(cls, pytd.Class): assert not cls.template # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in [ "__builtin__.type", "__builtin__.property" ]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(mycls, self.vm) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] elif isinstance(cls, pytd.Literal): return self.constant_to_value( self._get_literal_value(cls.value), subst, self.vm.root_cfg_node) else: return self.constant_to_value(cls, subst, self.vm.root_cfg_node) elif (isinstance(pyval, pytd.GenericType) and pyval.base_type.name == "typing.ClassVar"): param, = pyval.parameters return self.constant_to_value(param, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.GenericType): if isinstance(pyval.base_type, pytd.LateType): actual = self._load_late_type(pyval.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base = actual.cls else: assert isinstance(pyval.base_type, pytd.ClassType), pyval base = pyval.base_type.cls assert isinstance(base, pytd.Class), base base_cls = self.constant_to_value(base, subst, self.vm.root_cfg_node) if not isinstance(base_cls, mixin.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = list(range(len( pyval.parameters))) + [abstract_utils.T] parameters = pyval.parameters + (pytd.UnionType( pyval.parameters), ) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.CallableClass template = list(range(len( pyval.args))) + [abstract_utils.ARGS, abstract_utils.RET] parameters = pyval.args + (pytd_utils.JoinTypes( pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass if pyval.base_type.name == "typing.Generic": pyval_template = pyval.parameters else: pyval_template = base.template template = tuple(t.name for t in pyval_template) parameters = pyval.parameters assert (pyval.base_type.name == "typing.Generic" or len(parameters) <= len(template)) # Delay type parameter loading to handle recursive types. # See the ParameterizedClass.formal_type_parameters() property. type_parameters = abstract_utils.LazyFormalTypeParameters( template, parameters, subst) return abstract_class(base_cls, type_parameters, self.vm) elif isinstance(pyval, pytd.Literal): value = self.constant_to_value( self._get_literal_value(pyval.value), subst, self.vm.root_cfg_node) return abstract.LiteralClass(self.name_to_value("typing.Literal"), value, self.vm) elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.vm.root_cfg_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def constant_to_var(self, pyval, subst=None, node=None, source_sets=None, discard_concrete_values=False): """Convert a constant to a Variable. This converts a constant to a cfg.Variable. Unlike constant_to_value, it can handle things that need to be represented as a Variable with multiple possible values (i.e., a union type), like pytd.Function. Args: pyval: The Python constant to convert. Can be a PyTD definition or a builtin constant. subst: The current type parameters. node: The current CFG node. (For instances) source_sets: An iterator over instances of SourceSet (or just tuples). discard_concrete_values: Whether concrete values should be discarded from type parameters. Returns: A cfg.Variable. Raises: TypeParameterError: if conversion is attempted on a type parameter without a substitution. ValueError: if pytype is not of a known type. """ source_sets = source_sets or [[]] node = node or self.vm.root_cfg_node if isinstance(pyval, pytd.NothingType): return self.vm.program.NewVariable([], [], self.vm.root_cfg_node) elif isinstance(pyval, pytd.Alias): return self.constant_to_var(pyval.type, subst, node, source_sets, discard_concrete_values) elif isinstance(pyval, abstract_utils.AsInstance): cls = pyval.cls if isinstance(cls, pytd.AnythingType): return self.unsolvable.to_variable(node) elif (isinstance(pyval, abstract_utils.AsReturnValue) and isinstance(cls, pytd.NothingType)): return self.no_return.to_variable(node) var = self.vm.program.NewVariable() for t in pytd_utils.UnpackUnion(cls): if isinstance(t, pytd.TypeParameter): if not subst or t.full_name not in subst: raise self.TypeParameterError(t.full_name) else: for v in subst[t.full_name].bindings: for source_set in source_sets: var.AddBinding( self.get_maybe_abstract_instance(v.data) if discard_concrete_values else v.data, source_set + [v], node) elif isinstance(t, pytd.NothingType): pass else: value = self.constant_to_value( abstract_utils.AsInstance(t), subst, node) for source_set in source_sets: var.AddBinding(value, source_set, node) return var elif isinstance(pyval, pytd.Constant): return self.constant_to_var(abstract_utils.AsInstance(pyval.type), subst, node, source_sets, discard_concrete_values) result = self.constant_to_value(pyval, subst, node) if result is not None: return result.to_variable(node) # There might still be bugs on the abstract intepreter when it returns, # e.g. a list of values instead of a list of types: assert pyval.__class__ != cfg.Variable, pyval if pyval.__class__ == tuple: # This case needs to go at the end because many things are actually also # tuples. return self.build_tuple( self.vm.root_cfg_node, (self.constant_to_var( v, subst, node, source_sets, discard_concrete_values) for i, v in enumerate(pyval))) raise ValueError("Cannot convert {} to an abstract value".format( pyval.__class__))
def _convert(self, x, name, as_instance=False): pyval = self._parse_and_lookup(x, name) if as_instance: pyval = abstract_utils.AsInstance(pyval) return self.vm.convert.constant_to_value(pyval, {}, self.vm.root_cfg_node)