def tuple_to_value(self, node, content): """Create a VM tuple from the given sequence.""" content = tuple(content) # content might be a generator value = abstract.AbstractOrConcreteValue( content, self.tuple_type, self.vm, node) value.initialize_type_parameter(node, "T", self.build_content(node, content)) return value
def _get_attribute_computed(self, node, cls, name, valself, compute_function): """Call compute_function (if defined) to compute an attribute.""" assert isinstance(cls, (mixin.Class, abstract.AMBIGUOUS_OR_EMPTY)), cls if (valself and not isinstance(valself.data, abstract.Module) and self._computable(name)): attr_var = self._lookup_from_mro(node, cls, compute_function, valself, skip=self.vm.convert.object_type) if attr_var and attr_var.bindings: name_var = abstract.AbstractOrConcreteValue( name, self.vm.convert.str_type, self.vm).to_variable(node) return self.vm.call_function(node, attr_var, function.Args((name_var,))) return node, None
def _get_attribute_computed(self, node, cls, name, valself, valcls, compute_function): """Call compute_function (if defined) to compute an attribute.""" assert isinstance(cls, (abstract.Class, abstract.AMBIGUOUS_OR_EMPTY)) if (valself and not isinstance(valself.data, abstract.Module) and self._computable(name)): attr_var = self._lookup_from_mro( node, cls, compute_function, valself, valcls, skip=self.vm.convert.object_type.data[0]) if attr_var and attr_var.bindings: vm = self.vm # pytype: disable=attribute-error name_var = abstract.AbstractOrConcreteValue( name, vm.convert.str_type, vm, node).to_variable(node) return vm.call_function( node, attr_var, abstract.FunctionArgs((name_var,))) return node, None
def build_slice(self, node, start, stop, step=None): const_types = (int, type(None)) try: if start: start = abstract_utils.get_atomic_python_constant( start, const_types) if stop: stop = abstract_utils.get_atomic_python_constant( stop, const_types) if step: step = abstract_utils.get_atomic_python_constant( step, const_types) except abstract_utils.ConversionError: return self.primitive_class_instances[slice].to_variable(node) return abstract.AbstractOrConcreteValue(slice(start, stop, step), self.primitive_classes[slice], self.vm).to_variable(node)
def _constant_to_value(self, pyval, subst, get_node): """Create a AtomicAbstractValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if pyval.__class__ is str: # We use a subclass of str, compat.BytesPy3, to mark Python 3 # bytestrings, which are converted to abstract bytes instances. # compat.BytesType dispatches to this when appropriate. return abstract.AbstractOrConcreteValue(pyval, self.str_type, self.vm) elif isinstance(pyval, compat.UnicodeType): return abstract.AbstractOrConcreteValue(pyval, self.unicode_type, self.vm) elif isinstance(pyval, compat.BytesType): return abstract.AbstractOrConcreteValue(pyval, self.bytes_type, self.vm) elif isinstance(pyval, bool): return self.true if pyval is True else self.false elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.AbstractOrConcreteValue(pyval, self.int_type, self.vm) elif isinstance(pyval, compat.LongType): # long is aliased to int return self.primitive_class_instances[int] elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.AbstractOrConcreteValue( pyval, self.primitive_classes[types.CodeType], self.vm) elif pyval is super: return special_builtins.Super(self.vm) elif pyval is object: return special_builtins.Object(self.vm) elif pyval.__class__ is type: try: return self.name_to_value(self._type_to_name(pyval), subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.LateType): actual = self._load_late_type(pyval) return self._constant_to_value(actual, subst, get_node) elif isinstance(pyval, pytd.TypeDeclUnit): return self._create_module(pyval) elif isinstance(pyval, pytd.Module): mod = self.vm.loader.import_name(pyval.module_name) return self._create_module(mod) elif isinstance(pyval, pytd.Class): if pyval.name == "__builtin__.super": return self.vm.special_builtins["super"] elif pyval.name == "__builtin__.object": return self.object_type elif pyval.name == "types.ModuleType": return self.module_type elif pyval.name == "_importlib_modulespec.ModuleType": # Python 3's typeshed uses a stub file indirection to define ModuleType # even though it is exported via types.pyi. return self.module_type else: module, dot, base_name = pyval.name.rpartition(".") try: cls = abstract.PyTDClass(base_name, pyval, self.vm) except mro.MROError as e: self.vm.errorlog.mro_error(self.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module return cls elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.vm) for sig in pyval.signatures ] type_new = self.vm.lookup_builtin("__builtin__.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.vm) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): assert pyval.cls return self.constant_to_value(pyval.cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.NothingType): return self.empty elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif (isinstance(pyval, pytd.Constant) and isinstance(pyval.type, pytd.AnythingType)): # We allow "X = ... # type: Any" to declare X as a type. return self.unsolvable elif isinstance(pyval, pytd.FunctionType): return self.constant_to_value(pyval.function, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.vm.root_cfg_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.vm) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.vm.root_cfg_node) for c in pyval.constraints) bound = (pyval.bound and self.constant_to_value( pyval.bound, {}, self.vm.root_cfg_node)) return abstract.TypeParameter(pyval.name, self.vm, constraints=constraints, bound=bound) elif isinstance(pyval, abstract.AsInstance): cls = pyval.cls if isinstance(cls, pytd.LateType): actual = self._load_late_type(cls) if not isinstance(actual, pytd.ClassType): return self.unsolvable cls = actual.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if (isinstance(cls, pytd.GenericType) and cls.base_type.name == "typing.ClassVar"): param, = cls.parameters return self.constant_to_value(abstract.AsInstance(param), subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and cls.template): # If we're converting a generic Class, need to create a new instance of # it. See test_classes.testGenericReinstantiated. if isinstance(cls, pytd.Class): params = tuple(t.type_param.upper_value for t in cls.template) cls = pytd.GenericType(base_type=pytd.ClassType( cls.name, cls), parameters=params) if isinstance(cls.base_type, pytd.LateType): actual = self._load_late_type(cls.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base_cls = actual.cls else: assert isinstance(cls.base_type, pytd.ClassType) base_cls = cls.base_type.cls assert isinstance(base_cls, pytd.Class), base_cls if base_cls.name == "__builtin__.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.name not in subst: raise self.TypeParameterError(c.name) return self.merge_classes(get_node(), subst[c.name].data) else: return self.constant_to_value(c, subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.TupleType): content = tuple( self.constant_to_var(abstract.AsInstance(p), subst, get_node()) for p in cls.parameters) return abstract.Tuple(content, self.vm) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.vm.root_cfg_node) return abstract.Instance(clsval, self.vm) else: clsval = self.constant_to_value(base_cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(clsval, self.vm) num_params = len(cls.parameters) assert num_params <= len(base_cls.template) for i, formal in enumerate(base_cls.template): if i < num_params: node = get_node() p = self.constant_to_var( abstract.AsInstance(cls.parameters[i]), subst, node) else: # An omitted type parameter implies `Any`. node = self.vm.root_cfg_node p = self.unsolvable.to_variable(node) instance.merge_type_parameter(node, formal.name, p) return instance elif isinstance(cls, pytd.Class): assert not cls.template # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in [ "__builtin__.type", "__builtin__.property" ]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(mycls, self.vm) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] else: return self.constant_to_value(cls, subst, self.vm.root_cfg_node) elif (isinstance(pyval, pytd.GenericType) and pyval.base_type.name == "typing.ClassVar"): param, = pyval.parameters return self.constant_to_value(param, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.GenericType): if isinstance(pyval.base_type, pytd.LateType): actual = self._load_late_type(pyval.base_type) if not isinstance(actual, pytd.ClassType): return self.unsolvable base = actual.cls else: assert isinstance(pyval.base_type, pytd.ClassType) base = pyval.base_type.cls assert isinstance(base, pytd.Class), base base_cls = self.constant_to_value(base, subst, self.vm.root_cfg_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = list(range(len(pyval.parameters))) + [abstract.T] parameters = pyval.parameters + (pytd.UnionType( pyval.parameters), ) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.Callable template = list(range(len( pyval.args))) + [abstract.ARGS, abstract.RET] parameters = pyval.args + (pytd_utils.JoinTypes( pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass template = tuple(t.name for t in base.template) parameters = pyval.parameters assert (pyval.base_type.name == "typing.Generic" or len(parameters) <= len(template)) type_parameters = datatypes.LazyDict() for i, name in enumerate(template): if i < len(parameters): type_parameters.add_lazy_item(name, self.constant_to_value, parameters[i], subst, self.vm.root_cfg_node) else: type_parameters[name] = self.unsolvable return abstract_class(base_cls, type_parameters, self.vm) elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.vm.root_cfg_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def __init__(self, vm): super(Converter, self).__init__(vm) self.vm.convert = self # to make constant_to_value calls below work self.pytd_convert = output.Converter(vm) self._convert_cache = {} self._resolved_late_types = {} # performance cache # Initialize primitive_classes to empty to allow constant_to_value to run. self.primitive_classes = () # object_type is needed to initialize the primitive class values. self.object_type = self.constant_to_value(object) if self.vm.PY2: version_specific = [compat.UnicodeType] else: version_specific = [compat.BytesType] # Now fill primitive_classes with the real values using constant_to_value. self.primitive_classes = { v: self.constant_to_value(v) for v in [ int, float, str, object, frozenset, compat.NoneType, complex, bool, slice, types.CodeType, compat.EllipsisType, compat.OldStyleClassType, super ] + version_specific } self.primitive_class_names = [ self._type_to_name(x) for x in self.primitive_classes ] self.none = abstract.AbstractOrConcreteValue( None, self.primitive_classes[compat.NoneType], self.vm) self.true = abstract.AbstractOrConcreteValue( True, self.primitive_classes[bool], self.vm) self.false = abstract.AbstractOrConcreteValue( False, self.primitive_classes[bool], self.vm) self.ellipsis = abstract.AbstractOrConcreteValue( Ellipsis, self.primitive_classes[compat.EllipsisType], self.vm) self.primitive_class_instances = {} for name, cls in self.primitive_classes.items(): if name == compat.NoneType: # This is possible because all None instances are the same. # Without it pytype could not reason that "x is None" is always true, if # x is indeed None. instance = self.none elif name == compat.EllipsisType: instance = self.ellipsis else: instance = abstract.Instance(cls, self.vm) self.primitive_class_instances[name] = instance self._convert_cache[(abstract.Instance, cls.pytd_cls)] = instance self.none_type = self.primitive_classes[compat.NoneType] self.oldstyleclass_type = self.primitive_classes[ compat.OldStyleClassType] self.super_type = self.primitive_classes[super] self.str_type = self.primitive_classes[str] self.int_type = self.primitive_classes[int] self.unsolvable = abstract.Unsolvable(self.vm) self.empty = abstract.Empty(self.vm) self.no_return = typing_overlay.NoReturn(self.vm) self.list_type = self.constant_to_value(list) self.set_type = self.constant_to_value(set) self.dict_type = self.constant_to_value(dict) self.type_type = self.constant_to_value(type) self.module_type = self.constant_to_value(types.ModuleType) self.function_type = self.constant_to_value(types.FunctionType) self.tuple_type = self.constant_to_value(tuple) self.generator_type = self.constant_to_value(types.GeneratorType) self.iterator_type = self.constant_to_value(compat.IteratorType) self.bool_values = { True: self.true, False: self.false, None: self.primitive_class_instances[bool], } if self.vm.PY2: self.unicode_type = self.primitive_classes[compat.UnicodeType] self.bytes_type = self.str_type self.next_attr = "next" else: self.unicode_type = self.str_type self.bytes_type = self.primitive_classes[compat.BytesType] self.next_attr = "__next__"
def __init__(self, vm): self.vm = vm self.vm.convert = self # to make constant_to_value calls below work self.pytd_convert = output.Converter() self._convert_cache = {} self._resolved_late_types = {} # performance cache # Initialize primitive_classes to empty to allow constant_to_value to run. self.primitive_classes = () # object_type is needed to initialize the primitive class values. self.object_type = self.constant_to_value(object) # Now fill primitive_classes with the real values using constant_to_value. self.primitive_classes = { v: self.constant_to_value(v) for v in [ int, float, str, unicode, object, types.NoneType, complex, bool, slice, types.CodeType, types.EllipsisType, types.ClassType, super ] } self.primitive_class_names = [ x.__module__ + "." + x.__name__ for x in self.primitive_classes ] self.none = abstract.AbstractOrConcreteValue( None, self.primitive_classes[types.NoneType], self.vm) self.true = abstract.AbstractOrConcreteValue( True, self.primitive_classes[bool], self.vm) self.false = abstract.AbstractOrConcreteValue( False, self.primitive_classes[bool], self.vm) self.ellipsis = abstract.AbstractOrConcreteValue( Ellipsis, self.primitive_classes[types.EllipsisType], self.vm) self.primitive_class_instances = {} for name, cls in self.primitive_classes.items(): if name == types.NoneType: # This is possible because all None instances are the same. # Without it pytype could not reason that "x is None" is always true, if # x is indeed None. instance = self.none elif name == types.EllipsisType: instance = self.ellipsis else: instance = abstract.Instance(cls, self.vm) self.primitive_class_instances[name] = instance self._convert_cache[(abstract.Instance, cls.pytd_cls)] = instance self.none_type = self.primitive_classes[types.NoneType] self.oldstyleclass_type = self.primitive_classes[types.ClassType] self.super_type = self.primitive_classes[super] self.str_type = self.primitive_classes[str] self.int_type = self.primitive_classes[int] self.nothing = abstract.Nothing(self.vm) self.unsolvable = abstract.Unsolvable(self.vm) self.empty = abstract.Empty(self.vm) self.list_type = self.constant_to_value(list) self.set_type = self.constant_to_value(set) self.dict_type = self.constant_to_value(dict) self.type_type = self.constant_to_value(type) self.module_type = self.constant_to_value(types.ModuleType) self.function_type = self.constant_to_value(types.FunctionType) self.tuple_type = self.constant_to_value(tuple) self.generator_type = self.constant_to_value(types.GeneratorType) # TODO(dbaum): There isn't a types.IteratorType. This can probably be # based on typing.Iterator, but that will also require changes to # convert.py since that assumes all types can be looked up in # __builtin__. self.iterator_type = self.constant_to_value(types.ObjectType) self.bool_values = { True: self.true, False: self.false, None: self.primitive_class_instances[bool], }
def __init__(self, vm): self.vm = vm self.vm.convert = self # to make convert_constant calls below work self._convert_cache = {} # Initialize primitive_classes to empty to allow convert_constant to run self.primitive_classes = () # Now fill primitive_classes with the real values using convert_constant self.primitive_classes = {v: self.convert_constant(v.__name__, v) for v in [int, long, float, str, unicode, object, types.NoneType, complex, bool, slice, types.CodeType, types.EllipsisType, types.ClassType, super]} self.none = abstract.AbstractOrConcreteValue( None, self.primitive_classes[types.NoneType], self.vm, self.vm.root_cfg_node) self.true = abstract.AbstractOrConcreteValue( True, self.primitive_classes[bool], self.vm, self.vm.root_cfg_node) self.false = abstract.AbstractOrConcreteValue( False, self.primitive_classes[bool], self.vm, self.vm.root_cfg_node) self.ellipsis = abstract.AbstractOrConcreteValue( Ellipsis, self.primitive_classes[types.EllipsisType], self.vm, self.vm.root_cfg_node) self.primitive_class_instances = {} for name, clsvar in self.primitive_classes.items(): instance = abstract.Instance(clsvar, self.vm, self.vm.root_cfg_node) self.primitive_class_instances[name] = instance clsval, = clsvar.bindings self._convert_cache[(abstract.Instance, clsval.data.pytd_cls)] = instance self.primitive_class_instances[types.NoneType] = self.none self.none_type = self.primitive_classes[types.NoneType] self.object_type = self.primitive_classes[object] self.oldstyleclass_type = self.primitive_classes[types.ClassType] self.super_type = self.primitive_classes[super] self.str_type = self.primitive_classes[str] self.int_type = self.primitive_classes[int] self.nothing = abstract.Nothing(self.vm) self.unsolvable = abstract.Unsolvable(self.vm) self.tuple_type = self.convert_constant("tuple", tuple) self.list_type = self.convert_constant("list", list) self.set_type = self.convert_constant("set", set) self.dict_type = self.convert_constant("dict", dict) self.type_type = self.convert_constant("type", type) self.module_type = self.convert_constant("module", types.ModuleType) self.function_type = self.convert_constant( "function", types.FunctionType) self.generator_type = self.convert_constant( "generator", types.GeneratorType) # TODO(dbaum): There isn't a types.IteratorType. This can probably be # based on typing.Iterator, but that will also require changes to # convert.py since that assumes all types can be looked up in # __builtin__. self.iterator_type = self.convert_constant( "iterator", types.ObjectType) self.bool_values = { True: self.true, False: self.false, None: self.primitive_class_instances[bool], } self.undefined = self.vm.program.NewVariable("undefined")
def construct_constant_from_value(self, name, pyval, subst, node): """Create a AtomicAbstractValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: name: The name of this constant. Used for naming its attribute variables. pyval: The python or PyTD value to convert. subst: The current type parameters. node: The current CFG node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. """ if pyval is type: return abstract.SimpleAbstractValue(name, self.vm) elif isinstance(pyval, str): return abstract.AbstractOrConcreteValue( pyval, self.str_type, self.vm, node) elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.AbstractOrConcreteValue( pyval, self.int_type, self.vm, node) elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.AbstractOrConcreteValue( pyval, self.primitive_classes[types.CodeType], self.vm, node) elif pyval.__class__ in [types.FunctionType, types.ModuleType, types.GeneratorType, type]: try: pyclass = self.vm.vmbuiltins.Lookup("__builtin__." + pyval.__name__) return self.convert_constant_to_value(name, pyclass, subst, node) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.TypeDeclUnit): data = pyval.constants + pyval.classes + pyval.functions + pyval.aliases members = {val.name.rsplit(".")[-1]: val for val in data} return abstract.Module(self.vm, node, pyval.name, members) elif isinstance(pyval, pytd.Class): if "." in pyval.name: module, base_name = pyval.name.rsplit(".", 1) cls = abstract.PyTDClass(base_name, pyval, self.vm) cls.module = module else: cls = abstract.PyTDClass(name, pyval, self.vm) return cls elif isinstance(pyval, pytd.Function): signatures = [abstract.PyTDSignature(pyval.name, sig, self.vm) for sig in pyval.signatures] f = abstract.PyTDFunction( pyval.name, signatures, pyval.kind, self.vm, node) return f elif isinstance(pyval, pytd.ClassType): assert pyval.cls return self.convert_constant_to_value(pyval.name, pyval.cls, subst, node) elif isinstance(pyval, pytd.NothingType): return self.nothing elif isinstance(pyval, pytd.AnythingType): # TODO(kramm): This should be an Unsolveable. We don't need to solve this. return self._create_new_unknown_value("AnythingType") elif isinstance(pyval, pytd.FunctionType): return self.construct_constant_from_value( name, pyval.function, subst, node) elif isinstance(pyval, pytd.UnionType): return abstract.Union([ self.convert_constant_to_value(pytd.Print(t), t, subst, node) for t in pyval.type_list], self.vm) elif isinstance(pyval, pytd.TypeParameter): return abstract.TypeParameter(pyval.name, self.vm) elif isinstance(pyval, abstract.AsInstance): cls = pyval.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if isinstance(cls, pytd.Class): # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in ["__builtin__.type", "__builtin__.property"]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.convert_constant(cls.name, cls, subst, node) instance = abstract.Instance(mycls, self.vm, node) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] elif isinstance(cls, pytd.GenericType): assert isinstance(cls.base_type, pytd.ClassType) base_cls = cls.base_type.cls instance = abstract.Instance( self.convert_constant(base_cls.name, base_cls, subst, node), self.vm, node) for formal, actual in zip(base_cls.template, cls.parameters): p = self.convert_constant( repr(formal), abstract.AsInstance(actual), subst, node) instance.initialize_type_parameter(node, formal.name, p) return instance else: return self.convert_constant_to_value(name, cls, subst, node) elif isinstance(pyval, pytd.GenericType): assert isinstance(pyval.base_type, pytd.ClassType) type_parameters = utils.LazyDict() for param, value in zip(pyval.base_type.cls.template, pyval.parameters): type_parameters.add_lazy_item( param.name, self.convert_constant_to_value, param.name, value, subst, node) cls = self.convert_constant_to_value( pytd.Print(pyval.base_type), pyval.base_type.cls, subst, node) return abstract.ParameterizedClass(cls, type_parameters, self.vm) elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value(self.vm.root_cfg_node, [self.convert_constant("tuple[%d]" % i, item, subst, node) for i, item in enumerate(pyval)]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def new_tuple(*args): pyval = tuple(maybe_var(a) for a in args) return abstract.AbstractOrConcreteValue( pyval, self._vm.convert.tuple_type, self._vm, self._node)
def _constant_to_value(self, pyval, subst, get_node): """Create a AtomicAbstractValue that represents a python constant. This supports both constant from code constant pools and PyTD constants such as classes. This also supports builtin python objects such as int and float. Args: pyval: The python or PyTD value to convert. subst: The current type parameters. get_node: A getter function for the current node. Returns: A Value that represents the constant, or None if we couldn't convert. Raises: NotImplementedError: If we don't know how to convert a value. TypeParameterError: If we can't find a substitution for a type parameter. """ if isinstance(pyval, str): return abstract.AbstractOrConcreteValue(pyval, self.str_type, self.vm) elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH: # For small integers, preserve the actual value (for things like the # level in IMPORT_NAME). return abstract.AbstractOrConcreteValue(pyval, self.int_type, self.vm) elif isinstance(pyval, long): # long is aliased to int return self.primitive_class_instances[int] elif pyval.__class__ in self.primitive_classes: return self.primitive_class_instances[pyval.__class__] elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)): return abstract.AbstractOrConcreteValue( pyval, self.primitive_classes[types.CodeType], self.vm) elif pyval is super: return special_builtins.Super(self.vm) elif pyval is object: return special_builtins.Object(self.vm) elif pyval.__class__ is type: if pyval is types.FunctionType: classname = "typing.Callable" else: classname = "__builtin__." + pyval.__name__ try: return self.name_to_value(classname, subst) except (KeyError, AttributeError): log.debug("Failed to find pytd", exc_info=True) raise elif isinstance(pyval, pytd.TypeDeclUnit): data = (pyval.constants + pyval.type_params + pyval.classes + pyval.functions + pyval.aliases) members = {val.name.rsplit(".")[-1]: val for val in data} return abstract.Module(self.vm, pyval.name, members, pyval) elif isinstance(pyval, pytd.Class) and pyval.name == "__builtin__.super": return self.vm.special_builtins["super"] elif isinstance(pyval, pytd.Class) and pyval.name == "__builtin__.object": return self.object_type elif isinstance(pyval, pytd.Class) and pyval.name == "types.ModuleType": return self.module_type elif isinstance(pyval, pytd.Class): module, dot, base_name = pyval.name.rpartition(".") try: cls = abstract.PyTDClass(base_name, pyval, self.vm) except mro.MROError as e: self.vm.errorlog.mro_error(self.vm.frames, base_name, e.mro_seqs) cls = self.unsolvable else: if dot: cls.module = module return cls elif isinstance(pyval, pytd.ExternalFunction): module, _, name = pyval.name.partition(".") assert module == "__builtin__", "PYTHONCODE allowed only in __builtin__" return abstract.merge_values( self.vm.frame.f_globals.members[name].data, self.vm) elif isinstance(pyval, pytd.Function): signatures = [ abstract.PyTDSignature(pyval.name, sig, self.vm) for sig in pyval.signatures ] type_new = self.vm.lookup_builtin("__builtin__.type").Lookup( "__new__") if pyval is type_new: f_cls = special_builtins.TypeNew else: f_cls = abstract.PyTDFunction f = f_cls(pyval.name, signatures, pyval.kind, self.vm) f.is_abstract = pyval.is_abstract return f elif isinstance(pyval, pytd.ClassType): assert pyval.cls return self.constant_to_value(pyval.cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.NothingType): return self.nothing elif isinstance(pyval, pytd.AnythingType): return self.unsolvable elif isinstance(pyval, pytd.FunctionType): return self.constant_to_value(pyval.function, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.UnionType): options = [ self.constant_to_value(t, subst, self.vm.root_cfg_node) for t in pyval.type_list ] if len(options) > 1: return abstract.Union(options, self.vm) else: return options[0] elif isinstance(pyval, pytd.TypeParameter): constraints = tuple( self.constant_to_value(c, {}, self.vm.root_cfg_node) for c in pyval.constraints) bound = (pyval.bound and self.constant_to_value( pyval.bound, {}, self.vm.root_cfg_node)) return abstract.TypeParameter(pyval.name, self.vm, constraints=constraints, bound=bound) elif isinstance(pyval, abstract.AsInstance): cls = pyval.cls if isinstance(cls, pytd.ClassType): cls = cls.cls if isinstance(cls, pytd.Class): # This key is also used in __init__ key = (abstract.Instance, cls) if key not in self._convert_cache: if cls.name in [ "__builtin__.type", "__builtin__.property" ]: # An instance of "type" or of an anonymous property can be anything. instance = self._create_new_unknown_value("type") else: mycls = self.constant_to_value(cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(mycls, self.vm) instance.make_template_unsolvable( cls.template, self.vm.root_cfg_node) log.info("New pytd instance for %s: %r", cls.name, instance) self._convert_cache[key] = instance return self._convert_cache[key] elif isinstance(cls, pytd.GenericType): assert isinstance(cls.base_type, pytd.ClassType) base_cls = cls.base_type.cls if base_cls.name == "__builtin__.type": c, = cls.parameters if isinstance(c, pytd.TypeParameter): if not subst or c.name not in subst: raise self.TypeParameterError(c.name) return self.merge_classes(get_node(), subst[c.name].data) else: return self.constant_to_value(c, subst, self.vm.root_cfg_node) elif isinstance(cls, pytd.TupleType): content = tuple( self.constant_to_var(abstract.AsInstance(p), subst, get_node()) for p in cls.parameters) return abstract.Tuple(content, self.vm) elif isinstance(cls, pytd.CallableType): clsval = self.constant_to_value(cls, subst, self.vm.root_cfg_node) return abstract.Instance(clsval, self.vm) else: clsval = self.constant_to_value(base_cls, subst, self.vm.root_cfg_node) instance = abstract.Instance(clsval, self.vm) assert len(cls.parameters) <= len(base_cls.template) for formal, actual in zip(base_cls.template, cls.parameters): p = self.constant_to_var(abstract.AsInstance(actual), subst, self.vm.root_cfg_node) instance.initialize_type_parameter( get_node(), formal.name, p) return instance else: return self.constant_to_value(cls, subst, self.vm.root_cfg_node) elif isinstance(pyval, pytd.GenericType): assert isinstance(pyval.base_type, pytd.ClassType) base_cls = self.constant_to_value(pyval.base_type.cls, subst, self.vm.root_cfg_node) if not isinstance(base_cls, abstract.Class): # base_cls can be, e.g., an unsolvable due to an mro error. return self.unsolvable if isinstance(pyval, pytd.TupleType): abstract_class = abstract.TupleClass template = range(len(pyval.parameters)) + [abstract.T] parameters = pyval.parameters + (pytd.UnionType( pyval.parameters), ) elif isinstance(pyval, pytd.CallableType): abstract_class = abstract.Callable template = range(len( pyval.args)) + [abstract.ARGS, abstract.RET] parameters = pyval.args + (pytd_utils.JoinTypes( pyval.args), pyval.ret) else: abstract_class = abstract.ParameterizedClass template = tuple(t.name for t in pyval.base_type.cls.template) parameters = pyval.parameters assert (pyval.base_type.name == "typing.Generic" or len(parameters) <= len(template)) type_parameters = utils.LazyDict() for i, name in enumerate(template): if i < len(parameters): type_parameters.add_lazy_item(name, self.constant_to_value, parameters[i], subst, self.vm.root_cfg_node) else: type_parameters[name] = self.unsolvable return abstract_class(base_cls, type_parameters, self.vm) elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node return self.tuple_to_value([ self.constant_to_var(item, subst, self.vm.root_cfg_node) for i, item in enumerate(pyval) ]) else: raise NotImplementedError("Can't convert constant %s %r" % (type(pyval), pyval))
def __init__(self, vm): self.vm = vm self.vm.convert = self # to make constant_to_var calls below work self.pytd_convert = output.Converter() self._convert_cache = {} # Initialize primitive_classes to empty to allow constant_to_var to run self.primitive_classes = () # Now fill primitive_classes with the real values using constant_to_var self.primitive_classes = { v: self.constant_to_var(v.__name__, v) for v in [ int, float, str, unicode, object, types.NoneType, complex, bool, slice, types.CodeType, types.EllipsisType, types.ClassType, super ] } self.none = abstract.AbstractOrConcreteValue( None, self.primitive_classes[types.NoneType], self.vm, self.vm.root_cfg_node) self.true = abstract.AbstractOrConcreteValue( True, self.primitive_classes[bool], self.vm, self.vm.root_cfg_node) self.false = abstract.AbstractOrConcreteValue( False, self.primitive_classes[bool], self.vm, self.vm.root_cfg_node) self.ellipsis = abstract.AbstractOrConcreteValue( Ellipsis, self.primitive_classes[types.EllipsisType], self.vm, self.vm.root_cfg_node) self.primitive_class_instances = {} for name, clsvar in self.primitive_classes.items(): if name == types.NoneType: # This is possible because all None instances are the same. # Without it pytype could not reason that "x is None" is always true, if # x is indeed None. instance = self.none elif name == types.EllipsisType: instance = self.ellipsis else: instance = abstract.Instance(clsvar, self.vm, self.vm.root_cfg_node) self.primitive_class_instances[name] = instance clsval, = clsvar.bindings self._convert_cache[(abstract.Instance, clsval.data.pytd_cls)] = instance self.none_type = self.primitive_classes[types.NoneType] self.object_type = self.primitive_classes[object] self.oldstyleclass_type = self.primitive_classes[types.ClassType] self.super_type = self.primitive_classes[super] self.str_type = self.primitive_classes[str] self.int_type = self.primitive_classes[int] self.nothing = abstract.Nothing(self.vm) self.unsolvable = abstract.Unsolvable(self.vm) self.empty = abstract.Empty(self.vm) self.tuple_type = self.constant_to_var("tuple", tuple) self.list_type = self.constant_to_var("list", list) self.set_type = self.constant_to_var("set", set) self.frozenset_type = self.constant_to_var("frozenset", frozenset) self.dict_type = self.constant_to_var("dict", dict) self.type_type = self.constant_to_var("type", type) self.module_type = self.constant_to_var("module", types.ModuleType) self.function_type = self.constant_to_var("function", types.FunctionType) self.generator_type = self.constant_to_var("generator", types.GeneratorType) # TODO(dbaum): There isn't a types.IteratorType. This can probably be # based on typing.Iterator, but that will also require changes to # convert.py since that assumes all types can be looked up in # __builtin__. self.iterator_type = self.constant_to_var("iterator", types.ObjectType) self.bool_values = { True: self.true, False: self.false, None: self.primitive_class_instances[bool], } self.empty_type = self.empty.to_variable(self.vm.root_cfg_node) object_val, = self.object_type.data object_val.load_lazy_attribute("__new__") self.object_new, = object_val.members["__new__"].data self.typing_overlay = typing.TypingOverlay(self.vm, self.vm.root_cfg_node)