def make_class(self, node, f_locals): # If BuildClass.call() hits max depth, f_locals will be [unsolvable] # Since we don't support defining NamedTuple subclasses in a nested scope # anyway, we can just return unsolvable here to prevent a crash, and let the # invalid namedtuple error get raised later. if f_locals.data[0].isinstance_Unsolvable(): return node, self.vm.new_unsolvable(node) f_locals = abstract_utils.get_atomic_python_constant(f_locals) # retrieve __qualname__ to get the name of class name = f_locals["__qualname__"] # assemble the arguments that are compatible with NamedTupleFuncBuilder.call field_list = [] defaults = [] cls_locals = classgen.get_class_locals( abstract_utils.get_atomic_python_constant(name), allow_methods=True, ordering=classgen.Ordering.FIRST_ANNOTATE, vm=self.vm) for k, local in cls_locals.items(): assert local.typ if k in f_locals: defaults.append(f_locals[k]) k = self.vm.convert.constant_to_var(k, node=node) field_list.append(self.vm.convert.build_tuple( node, (k, local.typ))) anno = self.vm.convert.build_list(node, field_list) posargs = (name, anno) args = function.Args(posargs=posargs) node, cls_var = self.namedtuple.call(node, None, args) cls_val = abstract_utils.get_atomic_value(cls_var) if not isinstance(cls_val, abstract.Unsolvable): # set __new__.__defaults__ defaults = abstract.Tuple(tuple(defaults), self.vm).to_variable(node) node, new_attr = self.vm.attribute_handler.get_attribute( node, cls_val, "__new__") new_attr = abstract_utils.get_atomic_value(new_attr) node = self.vm.attribute_handler.set_attribute( node, new_attr, "__defaults__", defaults) # set the attribute without overriding special namedtuple attributes node, fields = self.vm.attribute_handler.get_attribute( node, cls_val, "_fields") fields = abstract_utils.get_atomic_python_constant(fields, tuple) fields = [ abstract_utils.get_atomic_python_constant(field, str) for field in fields ] for key in f_locals: if key in self._prohibited: self.vm.errorlog.not_writable(self.vm.frames, cls_val, key) if key not in self._special and key not in fields: node = self.vm.attribute_handler.set_attribute( node, cls_val, key, f_locals[key]) return node, cls_var
def test_getitem__concrete_index(self): t = abstract.Tuple((self._var, ), self._vm) index = self._vm.convert.constant_to_var(0) node, var = t.cls.data[0].getitem_slot(self._node, index) self.assertIs(node, self._node) self.assertIs(abstract.get_atomic_value(var), abstract.get_atomic_value(self._var))
def call(self, node, _, args): posargs = args.posargs if isinstance(args.namedargs, dict): namedargs = args.namedargs else: namedargs = self.vm.convert.value_to_constant(args.namedargs, dict) if namedargs and self.vm.python_version < (3, 6): errmsg = "Keyword syntax for NamedTuple is only supported in Python 3.6+" self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames, err_msg=errmsg) if namedargs and len(posargs) == 1: namedargs = [ abstract.Tuple((self.vm.convert.build_string(node, k), v), self.vm).to_variable(node) for k, v in namedargs.items() ] namedargs = abstract.List(namedargs, self.vm).to_variable(node) posargs += (namedargs, ) args = function.Args(posargs) elif namedargs: errmsg = ("Either list of fields or keywords can be provided to " "NamedTuple, not both") self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames, err_msg=errmsg) return self.namedtuple.call(node, None, args)
def test_call_with_all_args(self): f = self._make_func( param_names=("a", "b", "c"), varargs_name="arg", kwargs_name="kwarg", defaults=(self._vm.convert.build_int(self._vm.root_cfg_node),), annotations={ "a": self._vm.convert.str_type, "b": self._vm.convert.int_type, "c": self._vm.convert.int_type, "arg": self._vm.convert.primitive_classes[float], "kwarg": self._vm.convert.primitive_classes[bool] } ) posargs = (self._vm.convert.build_string(self._vm.root_cfg_node, "1"), self._vm.convert.build_int(self._vm.root_cfg_node)) float_inst = self._vm.convert.primitive_class_instances[float] stararg = abstract.Tuple((float_inst.to_variable(self._vm.root_cfg_node),), self._vm).to_variable(self._vm.root_cfg_node) namedargs = abstract.Dict(self._vm) kwarg = abstract.Dict(self._vm) kwarg.update(self._vm.root_cfg_node, {"x": self._vm.convert.build_bool(self._vm.root_cfg_node), "y": self._vm.convert.build_bool(self._vm.root_cfg_node)}) kwarg = kwarg.to_variable(self._vm.root_cfg_node) args = function.Args(posargs, namedargs, stararg, kwarg) f.call(self._vm.root_cfg_node, f, args)
def test_getitem__abstract_index(self): t = abstract.Tuple((self._var, ), self._vm) index = self._vm.convert.build_int(self._node) node, var = t.cls.data[0].getitem_slot(self._node, index) self.assertIs(node, self._node) self.assertIs(abstract.get_atomic_value(var), abstract.get_atomic_value(self._var))
def test_cmp_rel__prefix_unknown(self): tup1 = abstract.Tuple( (self._convert.constant_to_value(3).to_variable(self._node), self._convert.primitive_class_instances[int].to_variable( self._node)), self._vm) tup2 = self._convert.constant_to_value((3, 1)) for op in (slots.LT, slots.LE, slots.EQ, slots.NE, slots.GE, slots.GT): self.assertIsNone(compare.cmp_rel(self._vm, op, tup1, tup2)) self.assertIsNone(compare.cmp_rel(self._vm, op, tup2, tup1))
def test_to_type_with_view2(self): # to_type(<tuple (int or str,)>, view={0: str}) param1 = self._vm.convert.primitive_class_instances[int] param2 = self._vm.convert.primitive_class_instances[str] param_var = param1.to_variable(self._vm.root_cfg_node) str_binding = param_var.AddBinding(param2, [], self._vm.root_cfg_node) instance = abstract.Tuple((param_var,), self._vm) view = {param_var: str_binding} pytd_type = instance.to_type(self._vm.root_cfg_node, seen=None, view=view) self.assertEqual(pytd_type.parameters[0], pytd.NamedType("__builtin__.str"))
def test_call_with_bad_varargs(self): f = self._make_func( varargs_name="arg", annotations={"arg": self._vm.convert.str_type}) starargs = abstract.Tuple( (self._vm.convert.build_string(self._vm.root_node, ""), self._vm.convert.build_int(self._vm.root_node)), self._vm).to_variable(self._vm.root_node) args = function.Args(posargs=(), starargs=starargs) self.assertRaises(function.WrongArgTypes, f.call, self._vm.root_node, f, args)
def make_class(self, node, f_locals): f_locals = abstract_utils.get_atomic_python_constant(f_locals) # retrieve __qualname__ to get the name of class name = f_locals["__qualname__"] # retrieve __annotations__ to get the dict # with key-value pair of (variable, type) anno = f_locals.get("__annotations__", {}) if anno: anno = abstract_utils.get_atomic_value(anno) # assemble the arguments that are compatible with NamedTupleFuncBuilder.call field_list = [] defaults = [] for k, v in anno.items(): if k in f_locals: defaults.append(f_locals.get(k)) # TODO(ahxun): check if the value matches the declared type k = self.vm.convert.constant_to_var(k, node=node) field_list.append(self.vm.convert.build_tuple(node, (k, v))) anno = self.vm.convert.build_list(node, field_list) posargs = (name, anno) args = function.Args(posargs=posargs) node, cls_var = self.namedtuple.call(node, None, args) cls_val = abstract_utils.get_atomic_value(cls_var) if not isinstance(cls_val, abstract.Unsolvable): # set __new__.__defaults__ defaults = abstract.Tuple(tuple(defaults), self.vm).to_variable(node) node, new_attr = self.vm.attribute_handler.get_attribute( node, cls_val, "__new__") new_attr = abstract_utils.get_atomic_value(new_attr) node = self.vm.attribute_handler.set_attribute( node, new_attr, "__defaults__", defaults) # set the attribute without overriding special namedtuple attributes node, fields = self.vm.attribute_handler.get_attribute( node, cls_val, "_fields") fields = abstract_utils.get_atomic_python_constant(fields, tuple) fields = [ abstract_utils.get_atomic_python_constant(field, str) for field in fields ] for key in f_locals: if key in self._prohibited: self.vm.errorlog.not_writable(self.vm.frames, cls_val, key) if key not in self._special and key not in fields: node = self.vm.attribute_handler.set_attribute( node, cls_val, key, f_locals[key]) return node, cls_var
def test_call_with_multiple_varargs_bindings(self): f = self._make_func( varargs_name="arg", annotations={"arg": self._vm.convert.str_type}) arg = self._vm.program.NewVariable() arg.AddBinding(self._vm.convert.primitive_class_instances[str], [], self._vm.root_cfg_node) arg.AddBinding(self._vm.convert.primitive_class_instances[int], [], self._vm.root_cfg_node) starargs = abstract.Tuple((arg,), self._vm) starargs = starargs.to_variable(self._vm.root_cfg_node) args = function.Args(posargs=(), starargs=starargs) f.call(self._vm.root_cfg_node, f, args)
def test_call_with_varargs(self): f = self._make_func( varargs_name="arg", annotations={"arg": self._vm.convert.str_type, "return": self._vm.convert.str_type} ) starargs = abstract.Tuple( (self._vm.convert.build_string(self._vm.root_cfg_node, ""),), self._vm).to_variable(self._vm.root_cfg_node) args = function.Args(posargs=(), starargs=starargs) node, ret = f.call(self._vm.root_cfg_node, f, args) self.assertIs(node, self._vm.root_cfg_node) self.assertIs(ret.data[0].cls, self._vm.convert.str_type)
def test_change_defaults(self): f = self._make_func( param_names=("a", "b", "c"), defaults=(self._vm.convert.build_int(self._vm.root_node),)) args = function.Args( posargs=(self._vm.convert.build_int(self._vm.root_node), self._vm.convert.build_int(self._vm.root_node))) f.call(self._vm.root_node, f, args) new_defaults = abstract.Tuple((self._vm.convert.build_int( self._vm.root_node), self._vm.convert.build_int(self._vm.root_node)), self._vm).to_variable(self._vm.root_node) f.set_function_defaults(self._vm.root_node, new_defaults) f.call(self._vm.root_node, f, args) args = function.Args( posargs=(self._vm.convert.build_int(self._vm.root_node),)) f.call(self._vm.root_node, f, args)
def test_cmp_rel__prefix_not_equal(self): tup1 = abstract.Tuple( (self._convert.constant_to_value(3).to_variable(self._node), self._convert.constant_to_value(1).to_variable(self._node), self._convert.primitive_class_instances[int].to_variable( self._node)), self._vm) tup2 = self._convert.constant_to_value((4, 2)) self.assertIs(True, compare.cmp_rel(self._vm, slots.LT, tup1, tup2)) self.assertIs(False, compare.cmp_rel(self._vm, slots.LT, tup2, tup1)) self.assertIs(True, compare.cmp_rel(self._vm, slots.LE, tup1, tup2)) self.assertIs(False, compare.cmp_rel(self._vm, slots.LE, tup2, tup1)) self.assertIs(False, compare.cmp_rel(self._vm, slots.EQ, tup1, tup2)) self.assertIs(False, compare.cmp_rel(self._vm, slots.EQ, tup2, tup1)) self.assertIs(True, compare.cmp_rel(self._vm, slots.NE, tup1, tup2)) self.assertIs(True, compare.cmp_rel(self._vm, slots.NE, tup2, tup1)) self.assertIs(False, compare.cmp_rel(self._vm, slots.GE, tup1, tup2)) self.assertIs(True, compare.cmp_rel(self._vm, slots.GE, tup2, tup1)) self.assertIs(False, compare.cmp_rel(self._vm, slots.GT, tup1, tup2)) self.assertIs(True, compare.cmp_rel(self._vm, slots.GT, tup2, tup1))
def _constant_to_value(self, pyval, subst, get_node): """Create a AtomicAbstractValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if pyval.__class__ is str: # We use a subclass of str, compat.BytesPy3, to mark Python 3 # bytestrings, which are converted to abstract bytes instances. # compat.BytesType dispatches to this when appropriate. return abstract.AbstractOrConcreteValue(pyval, self.str_type, self.vm) elif isinstance(pyval, compat.UnicodeType): return abstract.AbstractOrConcreteValue(pyval, self.unicode_type, self.vm) elif isinstance(pyval, compat.BytesType): return abstract.AbstractOrConcreteValue(pyval, self.bytes_type, self.vm) elif isinstance(pyval, bool): return self.true if pyval is True else self.false elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.AbstractOrConcreteValue(pyval, self.int_type, self.vm) elif isinstance(pyval, compat.LongType): # long is aliased to int return self.primitive_class_instances[int] elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.AbstractOrConcreteValue( pyval, self.primitive_classes[types.CodeType], self.vm) elif pyval is super: return special_builtins.Super(self.vm) elif pyval is object: return special_builtins.Object(self.vm) elif pyval.__class__ is type: try: return self.name_to_value(self._type_to_name(pyval), subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.LateType): actual = self._load_late_type(pyval) return self._constant_to_value(actual, subst, get_node) elif isinstance(pyval, pytd.TypeDeclUnit): return self._create_module(pyval) elif isinstance(pyval, pytd.Module): mod = self.vm.loader.import_name(pyval.module_name) return self._create_module(mod) elif isinstance(pyval, pytd.Class): if pyval.name == "__builtin__.super": return self.vm.special_builtins["super"] elif pyval.name == "__builtin__.object": return self.object_type elif pyval.name == "types.ModuleType": return self.module_type elif pyval.name == "_importlib_modulespec.ModuleType": # Python 3's typeshed uses a stub file indirection to define ModuleType # even though it is exported via types.pyi. return self.module_type else: module, dot, base_name = pyval.name.rpartition(".") try: cls = abstract.PyTDClass(base_name, pyval, self.vm) except mro.MROError as e: self.vm.errorlog.mro_error(self.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module return cls elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.vm) for sig in pyval.signatures ] type_new = self.vm.lookup_builtin("__builtin__.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.vm) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): assert pyval.cls return self.constant_to_value(pyval.cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.NothingType): return self.empty elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.AnythingType)): # We allow "X = ... # type: Any" to declare X as a type. return self.unsolvable elif isinstance(pyval, pytd.FunctionType): return self.constant_to_value(pyval.function, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.vm.root_cfg_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.vm) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.vm.root_cfg_node) for c in pyval.constraints) bound = (pyval.bound and self.constant_to_value( pyval.bound, {}, self.vm.root_cfg_node)) return abstract.TypeParameter(pyval.name, self.vm, constraints=constraints, bound=bound) elif isinstance(pyval, abstract.AsInstance): cls = pyval.cls if isinstance(cls, pytd.LateType): actual = self._load_late_type(cls) if not isinstance(actual, pytd.ClassType): return self.unsolvable cls = actual.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if (isinstance(cls, pytd.GenericType) and cls.base_type.name == "typing.ClassVar"): param, = cls.parameters return self.constant_to_value(abstract.AsInstance(param), subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and cls.template): # If we're converting a generic Class, need to create a new instance of # it. See test_classes.testGenericReinstantiated. if isinstance(cls, pytd.Class): params = tuple(t.type_param.upper_value for t in cls.template) cls = pytd.GenericType(base_type=pytd.ClassType( cls.name, cls), parameters=params) if isinstance(cls.base_type, pytd.LateType): actual = self._load_late_type(cls.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base_cls = actual.cls else: assert isinstance(cls.base_type, pytd.ClassType) base_cls = cls.base_type.cls assert isinstance(base_cls, pytd.Class), base_cls if base_cls.name == "__builtin__.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.name not in subst: raise self.TypeParameterError(c.name) return self.merge_classes(get_node(), subst[c.name].data) else: return self.constant_to_value(c, subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.TupleType): content = tuple( self.constant_to_var(abstract.AsInstance(p), subst, get_node()) for p in cls.parameters) return abstract.Tuple(content, self.vm) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.vm.root_cfg_node) return abstract.Instance(clsval, self.vm) else: clsval = self.constant_to_value(base_cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(clsval, self.vm) num_params = len(cls.parameters) assert num_params <= len(base_cls.template) for i, formal in enumerate(base_cls.template): if i < num_params: node = get_node() p = self.constant_to_var( abstract.AsInstance(cls.parameters[i]), subst, node) else: # An omitted type parameter implies `Any`. node = self.vm.root_cfg_node p = self.unsolvable.to_variable(node) instance.merge_type_parameter(node, formal.name, p) return instance elif isinstance(cls, pytd.Class): assert not cls.template # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in [ "__builtin__.type", "__builtin__.property" ]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(mycls, self.vm) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] else: return self.constant_to_value(cls, subst, self.vm.root_cfg_node) elif (isinstance(pyval, pytd.GenericType) and pyval.base_type.name == "typing.ClassVar"): param, = pyval.parameters return self.constant_to_value(param, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.GenericType): if isinstance(pyval.base_type, pytd.LateType): actual = self._load_late_type(pyval.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base = actual.cls else: assert isinstance(pyval.base_type, pytd.ClassType) base = pyval.base_type.cls assert isinstance(base, pytd.Class), base base_cls = self.constant_to_value(base, subst, self.vm.root_cfg_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = list(range(len(pyval.parameters))) + [abstract.T] parameters = pyval.parameters + (pytd.UnionType( pyval.parameters), ) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.Callable template = list(range(len( pyval.args))) + [abstract.ARGS, abstract.RET] parameters = pyval.args + (pytd_utils.JoinTypes( pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass template = tuple(t.name for t in base.template) parameters = pyval.parameters assert (pyval.base_type.name == "typing.Generic" or len(parameters) <= len(template)) type_parameters = datatypes.LazyDict() for i, name in enumerate(template): if i < len(parameters): type_parameters.add_lazy_item(name, self.constant_to_value, parameters[i], subst, self.vm.root_cfg_node) else: type_parameters[name] = self.unsolvable return abstract_class(base_cls, type_parameters, self.vm) elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.vm.root_cfg_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def test_compatible_with__empty(self): t = abstract.Tuple((), self._vm) self.assertFalsy(t)
def tuple_to_value(self, content): """Create a VM tuple from the given sequence.""" content = tuple(content) # content might be a generator value = abstract.Tuple(content, self.vm) return value
def _constant_to_value(self, pyval, subst, get_node): """Create a AtomicAbstractValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if isinstance(pyval, str): return abstract.AbstractOrConcreteValue(pyval, self.str_type, self.vm) elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.AbstractOrConcreteValue(pyval, self.int_type, self.vm) elif isinstance(pyval, long): # long is aliased to int return self.primitive_class_instances[int] elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.AbstractOrConcreteValue( pyval, self.primitive_classes[types.CodeType], self.vm) elif pyval is super: return special_builtins.Super(self.vm) elif pyval is object: return special_builtins.Object(self.vm) elif pyval.__class__ is type: if pyval is types.FunctionType: classname = "typing.Callable" else: classname = "__builtin__." + pyval.__name__ try: return self.name_to_value(classname, subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.TypeDeclUnit): data = (pyval.constants + pyval.type_params + pyval.classes + pyval.functions + pyval.aliases) members = {val.name.rsplit(".")[-1]: val for val in data} return abstract.Module(self.vm, pyval.name, members, pyval) elif isinstance(pyval, pytd.Class) and pyval.name == "__builtin__.super": return self.vm.special_builtins["super"] elif isinstance(pyval, pytd.Class) and pyval.name == "__builtin__.object": return self.object_type elif isinstance(pyval, pytd.Class) and pyval.name == "types.ModuleType": return self.module_type elif isinstance(pyval, pytd.Class): module, dot, base_name = pyval.name.rpartition(".") try: cls = abstract.PyTDClass(base_name, pyval, self.vm) except mro.MROError as e: self.vm.errorlog.mro_error(self.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module return cls elif isinstance(pyval, pytd.ExternalFunction): module, _, name = pyval.name.partition(".") assert module == "__builtin__", "PYTHONCODE allowed only in __builtin__" return abstract.merge_values( self.vm.frame.f_globals.members[name].data, self.vm) elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.vm) for sig in pyval.signatures ] type_new = self.vm.lookup_builtin("__builtin__.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.vm) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): assert pyval.cls return self.constant_to_value(pyval.cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.NothingType): return self.nothing elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif isinstance(pyval, pytd.FunctionType): return self.constant_to_value(pyval.function, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.vm.root_cfg_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.vm) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.vm.root_cfg_node) for c in pyval.constraints) bound = (pyval.bound and self.constant_to_value( pyval.bound, {}, self.vm.root_cfg_node)) return abstract.TypeParameter(pyval.name, self.vm, constraints=constraints, bound=bound) elif isinstance(pyval, abstract.AsInstance): cls = pyval.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if isinstance(cls, pytd.Class): # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in [ "__builtin__.type", "__builtin__.property" ]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(mycls, self.vm) instance.make_template_unsolvable( cls.template, self.vm.root_cfg_node) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] elif isinstance(cls, pytd.GenericType): assert isinstance(cls.base_type, pytd.ClassType) base_cls = cls.base_type.cls if base_cls.name == "__builtin__.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.name not in subst: raise self.TypeParameterError(c.name) return self.merge_classes(get_node(), subst[c.name].data) else: return self.constant_to_value(c, subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.TupleType): content = tuple( self.constant_to_var(abstract.AsInstance(p), subst, get_node()) for p in cls.parameters) return abstract.Tuple(content, self.vm) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.vm.root_cfg_node) return abstract.Instance(clsval, self.vm) else: clsval = self.constant_to_value(base_cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(clsval, self.vm) assert len(cls.parameters) <= len(base_cls.template) for formal, actual in zip(base_cls.template, cls.parameters): p = self.constant_to_var(abstract.AsInstance(actual), subst, self.vm.root_cfg_node) instance.initialize_type_parameter( get_node(), formal.name, p) return instance else: return self.constant_to_value(cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.GenericType): assert isinstance(pyval.base_type, pytd.ClassType) base_cls = self.constant_to_value(pyval.base_type.cls, subst, self.vm.root_cfg_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = range(len(pyval.parameters)) + [abstract.T] parameters = pyval.parameters + (pytd.UnionType( pyval.parameters), ) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.Callable template = range(len( pyval.args)) + [abstract.ARGS, abstract.RET] parameters = pyval.args + (pytd_utils.JoinTypes( pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass template = tuple(t.name for t in pyval.base_type.cls.template) parameters = pyval.parameters assert (pyval.base_type.name == "typing.Generic" or len(parameters) <= len(template)) type_parameters = utils.LazyDict() for i, name in enumerate(template): if i < len(parameters): type_parameters.add_lazy_item(name, self.constant_to_value, parameters[i], subst, self.vm.root_cfg_node) else: type_parameters[name] = self.unsolvable return abstract_class(base_cls, type_parameters, self.vm) elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.vm.root_cfg_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def test_compatible_with__not_empty(self): t = abstract.Tuple((self._var, ), self._vm) self.assertTruthy(t)
def build_version_info(name, vm): del name tup = tuple(vm.convert.constant_to_var(i) for i in vm.python_version) return abstract.Tuple(tup, vm)
def _build_namedtuple(self, name, field_names, field_types, late_annots, node): # Build an InterpreterClass representing the namedtuple. if field_types: # TODO(mdemello): Fix this to support late types. field_types_union = abstract.Union(field_types, self.vm) else: field_types_union = self.vm.convert.none_type members = { n: t.instantiate(node) for n, t in moves.zip(field_names, field_types) } # collections.namedtuple has: __dict__, __slots__ and _fields. # typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults. # __slots__ and _fields are tuples containing the names of the fields. slots = tuple( self.vm.convert.build_string(node, f) for f in field_names) members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node) members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node) # __dict__ and _field_defaults are both collections.OrderedDicts that map # field names (strings) to objects of the field types. ordered_dict_cls = self.vm.convert.name_to_value( "collections.OrderedDict", ast=self.collections_ast) # In Python 2, keys can be `str` or `unicode`; support both. # In Python 3, `str_type` and `unicode_type` are the same. field_keys_union = abstract.Union( [self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm) # Normally, we would use abstract_utils.K and abstract_utils.V, but # collections.pyi doesn't conform to that standard. field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, { "K": field_keys_union, "V": field_types_union }, self.vm) members["__dict__"] = field_dict_cls.instantiate(node) members["_field_defaults"] = field_dict_cls.instantiate(node) # _field_types and __annotations__ are both collections.OrderedDicts # that map field names (strings) to the types of the fields. field_types_cls = abstract.ParameterizedClass( ordered_dict_cls, { "K": field_keys_union, "V": self.vm.convert.type_type }, self.vm) members["_field_types"] = field_types_cls.instantiate(node) members["__annotations__"] = field_types_cls.instantiate(node) # __new__ # We set the bound on this TypeParameter later. This gives __new__ the # signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same # signature that visitor.CreateTypeParametersForSignatures would create. # This allows subclasses of the NamedTuple to get the correct type from # their constructors. cls_type_param = abstract.TypeParameter( visitors.CreateTypeParametersForSignatures.PREFIX + name, self.vm, bound=None) cls_type = abstract.ParameterizedClass( self.vm.convert.type_type, {abstract_utils.T: cls_type_param}, self.vm) # Use late annotations as field types if they exist. params = [ Param(n, late_annots.get(n, t)) for n, t in moves.zip(field_names, field_types) ] members["__new__"] = overlay_utils.make_method( self.vm, node, name="__new__", self_param=Param("cls", cls_type), params=params, return_type=cls_type_param, ) # __init__ members["__init__"] = overlay_utils.make_method(self.vm, node, name="__init__", varargs=Param("args"), kwargs=Param("kwargs")) # _make # _make is a classmethod, so it needs to be wrapped by # specialibuiltins.ClassMethodInstance. # Like __new__, it uses the _Tname TypeVar. sized_cls = self.vm.convert.name_to_value("typing.Sized") iterable_type = abstract.ParameterizedClass( self.vm.convert.name_to_value("typing.Iterable"), {abstract_utils.T: field_types_union}, self.vm) cls_type = abstract.ParameterizedClass( self.vm.convert.type_type, {abstract_utils.T: cls_type_param}, self.vm) len_type = abstract.CallableClass( self.vm.convert.name_to_value("typing.Callable"), { 0: sized_cls, abstract_utils.ARGS: sized_cls, abstract_utils.RET: self.vm.convert.int_type }, self.vm) params = [ Param("iterable", iterable_type), Param("new").unsolvable(self.vm, node), Param("len", len_type).unsolvable(self.vm, node) ] make = overlay_utils.make_method(self.vm, node, name="_make", params=params, self_param=Param("cls", cls_type), return_type=cls_type_param) make_args = function.Args(posargs=(make, )) _, members["_make"] = self.vm.special_builtins["classmethod"].call( node, None, make_args) # _replace # Like __new__, it uses the _Tname TypeVar. We have to annotate the `self` # param to make sure the TypeVar is substituted correctly. members["_replace"] = overlay_utils.make_method( self.vm, node, name="_replace", self_param=Param("self", cls_type_param), return_type=cls_type_param, kwargs=Param("kwds", field_types_union)) # __getnewargs__ getnewargs_tuple_params = dict( tuple(enumerate(field_types)) + ((abstract_utils.T, field_types_union), )) getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type, getnewargs_tuple_params, self.vm) members["__getnewargs__"] = overlay_utils.make_method( self.vm, node, name="__getnewargs__", return_type=getnewargs_tuple) # __getstate__ members["__getstate__"] = overlay_utils.make_method( self.vm, node, name="__getstate__") # _asdict members["_asdict"] = overlay_utils.make_method( self.vm, node, name="_asdict", return_type=field_dict_cls) # Finally, make the class. cls_dict = abstract.Dict(self.vm) cls_dict.update(node, members) if name.__class__ is compat.UnicodeType: # Unicode values should be ASCII. name = compat.native_str(name.encode("ascii")) node, cls_var = self.vm.make_class( node=node, name_var=self.vm.convert.build_string(node, name), bases=[self.vm.convert.tuple_type.to_variable(node)], class_dict_var=cls_dict.to_variable(node), cls_var=None) cls = cls_var.data[0] # Now that the class has been made, we can complete the TypeParameter used # by __new__, _make and _replace. cls_type_param.bound = cls # Add late annotations to the new class if late_annots: cls.late_annotations = late_annots self.vm.classes_with_late_annotations.append(cls) return node, cls_var
def _build_namedtuple(self, name, field_names, field_types, node): # Build an InterpreterClass representing the namedtuple. if field_types: field_types_union = abstract.Union(field_types, self.vm) else: field_types_union = self.vm.convert.none_type members = { n: t.instantiate(node) for n, t in moves.zip(field_names, field_types) } # collections.namedtuple has: __dict__, __slots__ and _fields. # typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults. # __slots__ and _fields are tuples containing the names of the fields. slots = tuple( self.vm.convert.build_string(node, f) for f in field_names) members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node) members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node) # __dict__ and _field_defaults are both collections.OrderedDicts that map # field names (strings) to objects of the field types. ordered_dict_cls = self.vm.convert.name_to_value( "collections.OrderedDict", ast=self.collections_ast) # In Python 2, keys can be `str` or `unicode`; support both. # In Python 3, `str_type` and `unicode_type` are the same. field_keys_union = abstract.Union( [self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm) # Normally, we would use abstract_utils.K and abstract_utils.V, but # collections.pyi doesn't conform to that standard. field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, { "K": field_keys_union, "V": field_types_union }, self.vm) members["__dict__"] = field_dict_cls.instantiate(node) members["_field_defaults"] = field_dict_cls.instantiate(node) # _field_types and __annotations__ are both collections.OrderedDicts # that map field names (strings) to the types of the fields. field_types_cls = abstract.ParameterizedClass( ordered_dict_cls, { "K": field_keys_union, "V": self.vm.convert.type_type }, self.vm) members["_field_types"] = field_types_cls.instantiate(node) members["__annotations__"] = field_types_cls.instantiate(node) # __new__ new_annots = {} new_lates = {} for (n, t) in moves.zip(field_names, field_types): # We don't support late annotations yet, but once we do, they'll show up # as LateAnnotation objects to be stored in new_lates. new_annots[n] = t # We set the bound on this TypeParameter later. This gives __new__ the # signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same # signature that visitor.CreateTypeParametersForSignatures would create. # This allows subclasses of the NamedTuple to get the correct type from # their constructors. cls_type_param = abstract.TypeParameter( visitors.CreateTypeParametersForSignatures.PREFIX + name, self.vm, bound=None) new_annots["cls"] = abstract.ParameterizedClass( self.vm.convert.type_type, {abstract_utils.T: cls_type_param}, self.vm) new_annots["return"] = cls_type_param members["__new__"] = abstract.SimpleFunction( name="__new__", param_names=("cls", ) + tuple(field_names), varargs_name=None, kwonly_params=(), kwargs_name=None, defaults={}, annotations=new_annots, late_annotations=new_lates, vm=self.vm).to_variable(node) # __init__ members["__init__"] = abstract.SimpleFunction( name="__init__", param_names=("self", ), varargs_name="args", kwonly_params=(), kwargs_name="kwargs", defaults={}, annotations={}, late_annotations={}, vm=self.vm).to_variable(node) # _make # _make is a classmethod, so it needs to be wrapped by # specialibuiltins.ClassMethodInstance. # Like __new__, it uses the _Tname TypeVar. sized_cls = self.vm.convert.name_to_value("typing.Sized") iterable_type = abstract.ParameterizedClass( self.vm.convert.name_to_value("typing.Iterable"), {abstract_utils.T: field_types_union}, self.vm) make = abstract.SimpleFunction( name="_make", param_names=("cls", "iterable", "new", "len"), varargs_name=None, kwonly_params=(), kwargs_name=None, defaults={ "new": self.vm.convert.unsolvable.to_variable(node), "len": self.vm.convert.unsolvable.to_variable(node) }, annotations={ "cls": abstract.ParameterizedClass(self.vm.convert.type_type, {abstract_utils.T: cls_type_param}, self.vm), "iterable": iterable_type, "new": self.vm.convert.unsolvable, "len": abstract.Callable( self.vm.convert.name_to_value("typing.Callable"), { 0: sized_cls, abstract_utils.ARGS: sized_cls, abstract_utils.RET: self.vm.convert.int_type }, self.vm), "return": cls_type_param }, late_annotations={}, vm=self.vm).to_variable(node) make_args = function.Args(posargs=(make, )) _, members["_make"] = self.vm.special_builtins["classmethod"].call( node, None, make_args) # _replace # Like __new__, it uses the _Tname TypeVar. We have to annotate the `self` # param to make sure the TypeVar is substituted correctly. members["_replace"] = abstract.SimpleFunction( name="_replace", param_names=("self", ), varargs_name=None, kwonly_params=(), kwargs_name="kwds", defaults={}, annotations={ "self": cls_type_param, "kwds": field_types_union, "return": cls_type_param }, late_annotations={}, vm=self.vm).to_variable(node) # __getnewargs__ getnewargs_tuple_params = dict( tuple(enumerate(field_types)) + ((abstract_utils.T, field_types_union), )) getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type, getnewargs_tuple_params, self.vm) members["__getnewargs__"] = abstract.SimpleFunction( name="__getnewargs__", param_names=("self", ), varargs_name=None, kwonly_params=(), kwargs_name=None, defaults={}, annotations={ "return": getnewargs_tuple }, late_annotations={}, vm=self.vm).to_variable(node) # __getstate__ members["__getstate__"] = abstract.SimpleFunction( name="__getstate__", param_names=("self", ), varargs_name=None, kwonly_params=(), kwargs_name=None, defaults={}, annotations={}, late_annotations={}, vm=self.vm).to_variable(node) # _asdict members["_asdict"] = abstract.SimpleFunction( name="_asdict", param_names=("self", ), varargs_name=None, kwonly_params=(), kwargs_name=None, defaults={}, annotations={ "return": field_dict_cls }, late_annotations={}, vm=self.vm).to_variable(node) # Finally, make the class. abs_membs = abstract.Dict(self.vm) abs_membs.update(node, members) cls_var = self.vm.make_class( node=node, name_var=self.vm.convert.build_string(node, name), bases=[self.vm.convert.tuple_type.to_variable(node)], class_dict_var=abs_membs.to_variable(node), cls_var=None) # Now that the class has been made, we can complete the TypeParameter used # by __new__, _make and _replace. cls_type_param.bound = cls_var.data[0] return cls_var
def test_compatible_with__not_empty(self): t = abstract.Tuple((self._var, ), self._vm, self._node) self.assertIs(True, t.compatible_with(True)) self.assertIs(False, t.compatible_with(False))
def test_compatible_with__not_empty(self): t = abstract.Tuple((self._var, ), self._vm) self.assertIs(True, compare.compatible_with(t, True)) self.assertIs(False, compare.compatible_with(t, False))
def test_compatible_with__empty(self): t = abstract.Tuple((), self._vm) self.assertIs(False, t.compatible_with(True)) self.assertIs(True, t.compatible_with(False))
def test_getitem__concrete_index(self): t = abstract.Tuple((self._var, ), self._vm, self._node) index = self._vm.convert.constant_to_var("index", 0) node, var = t.getitem_slot(self._node, index) self.assertIs(node, self._node) self.assertIs(var, self._var)