def tuple_to_value(self, node, content):
"""Create a VM tuple from the given sequence."""
content = tuple(content) # content might be a generator
value = abstract.AbstractOrConcreteValue(
content, self.tuple_type, self.vm, node)
value.initialize_type_parameter(node, "T",
self.build_content(node, content))
return value
def _get_attribute_computed(self, node, cls, name, valself, compute_function):
"""Call compute_function (if defined) to compute an attribute."""
assert isinstance(cls, (mixin.Class, abstract.AMBIGUOUS_OR_EMPTY)), cls
if (valself and not isinstance(valself.data, abstract.Module) and
self._computable(name)):
attr_var = self._lookup_from_mro(node, cls, compute_function, valself,
skip=self.vm.convert.object_type)
if attr_var and attr_var.bindings:
name_var = abstract.AbstractOrConcreteValue(
name, self.vm.convert.str_type, self.vm).to_variable(node)
return self.vm.call_function(node, attr_var, function.Args((name_var,)))
return node, None
def _get_attribute_computed(self, node, cls, name, valself, valcls,
compute_function):
"""Call compute_function (if defined) to compute an attribute."""
assert isinstance(cls, (abstract.Class, abstract.AMBIGUOUS_OR_EMPTY))
if (valself and not isinstance(valself.data, abstract.Module) and
self._computable(name)):
attr_var = self._lookup_from_mro(
node, cls, compute_function, valself, valcls,
skip=self.vm.convert.object_type.data[0])
if attr_var and attr_var.bindings:
vm = self.vm # pytype: disable=attribute-error
name_var = abstract.AbstractOrConcreteValue(
name, vm.convert.str_type, vm, node).to_variable(node)
return vm.call_function(
node, attr_var, abstract.FunctionArgs((name_var,)))
return node, None
def build_slice(self, node, start, stop, step=None):
const_types = (int, type(None))
try:
if start:
start = abstract_utils.get_atomic_python_constant(
start, const_types)
if stop:
stop = abstract_utils.get_atomic_python_constant(
stop, const_types)
if step:
step = abstract_utils.get_atomic_python_constant(
step, const_types)
except abstract_utils.ConversionError:
return self.primitive_class_instances[slice].to_variable(node)
return abstract.AbstractOrConcreteValue(slice(start, stop, step),
self.primitive_classes[slice],
self.vm).to_variable(node)
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if pyval.__class__ is str:
# We use a subclass of str, compat.BytesPy3, to mark Python 3
# bytestrings, which are converted to abstract bytes instances.
# compat.BytesType dispatches to this when appropriate.
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, compat.UnicodeType):
return abstract.AbstractOrConcreteValue(pyval, self.unicode_type,
self.vm)
elif isinstance(pyval, compat.BytesType):
return abstract.AbstractOrConcreteValue(pyval, self.bytes_type,
self.vm)
elif isinstance(pyval, bool):
return self.true if pyval is True else self.false
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, compat.LongType):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.vm.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif pyval.name == "__builtin__.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
else:
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant)
and isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if (isinstance(cls, pytd.GenericType)
and cls.base_type.name == "typing.ClassVar"):
param, = cls.parameters
return self.constant_to_value(abstract.AsInstance(param),
subst, self.vm.root_cfg_node)
elif isinstance(cls,
pytd.GenericType) or (isinstance(cls, pytd.Class)
and cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value
for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(
cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract.AsInstance(cls.parameters[i]), subst,
node)
else:
# An omitted type parameter implies `Any`.
node = self.vm.root_cfg_node
p = self.unsolvable.to_variable(node)
instance.merge_type_parameter(node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif (isinstance(pyval, pytd.GenericType)
and pyval.base_type.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType)
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(pyval.parameters))) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = list(range(len(
pyval.args))) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in base.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = datatypes.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def __init__(self, vm):
super(Converter, self).__init__(vm)
self.vm.convert = self # to make constant_to_value calls below work
self.pytd_convert = output.Converter(vm)
self._convert_cache = {}
self._resolved_late_types = {} # performance cache
# Initialize primitive_classes to empty to allow constant_to_value to run.
self.primitive_classes = ()
# object_type is needed to initialize the primitive class values.
self.object_type = self.constant_to_value(object)
if self.vm.PY2:
version_specific = [compat.UnicodeType]
else:
version_specific = [compat.BytesType]
# Now fill primitive_classes with the real values using constant_to_value.
self.primitive_classes = {
v: self.constant_to_value(v)
for v in [
int, float, str, object, frozenset, compat.NoneType, complex,
bool, slice, types.CodeType, compat.EllipsisType,
compat.OldStyleClassType, super
] + version_specific
}
self.primitive_class_names = [
self._type_to_name(x) for x in self.primitive_classes
]
self.none = abstract.AbstractOrConcreteValue(
None, self.primitive_classes[compat.NoneType], self.vm)
self.true = abstract.AbstractOrConcreteValue(
True, self.primitive_classes[bool], self.vm)
self.false = abstract.AbstractOrConcreteValue(
False, self.primitive_classes[bool], self.vm)
self.ellipsis = abstract.AbstractOrConcreteValue(
Ellipsis, self.primitive_classes[compat.EllipsisType], self.vm)
self.primitive_class_instances = {}
for name, cls in self.primitive_classes.items():
if name == compat.NoneType:
# This is possible because all None instances are the same.
# Without it pytype could not reason that "x is None" is always true, if
# x is indeed None.
instance = self.none
elif name == compat.EllipsisType:
instance = self.ellipsis
else:
instance = abstract.Instance(cls, self.vm)
self.primitive_class_instances[name] = instance
self._convert_cache[(abstract.Instance, cls.pytd_cls)] = instance
self.none_type = self.primitive_classes[compat.NoneType]
self.oldstyleclass_type = self.primitive_classes[
compat.OldStyleClassType]
self.super_type = self.primitive_classes[super]
self.str_type = self.primitive_classes[str]
self.int_type = self.primitive_classes[int]
self.unsolvable = abstract.Unsolvable(self.vm)
self.empty = abstract.Empty(self.vm)
self.no_return = typing_overlay.NoReturn(self.vm)
self.list_type = self.constant_to_value(list)
self.set_type = self.constant_to_value(set)
self.dict_type = self.constant_to_value(dict)
self.type_type = self.constant_to_value(type)
self.module_type = self.constant_to_value(types.ModuleType)
self.function_type = self.constant_to_value(types.FunctionType)
self.tuple_type = self.constant_to_value(tuple)
self.generator_type = self.constant_to_value(types.GeneratorType)
self.iterator_type = self.constant_to_value(compat.IteratorType)
self.bool_values = {
True: self.true,
False: self.false,
None: self.primitive_class_instances[bool],
}
if self.vm.PY2:
self.unicode_type = self.primitive_classes[compat.UnicodeType]
self.bytes_type = self.str_type
self.next_attr = "next"
else:
self.unicode_type = self.str_type
self.bytes_type = self.primitive_classes[compat.BytesType]
self.next_attr = "__next__"
def __init__(self, vm):
self.vm = vm
self.vm.convert = self # to make constant_to_value calls below work
self.pytd_convert = output.Converter()
self._convert_cache = {}
self._resolved_late_types = {} # performance cache
# Initialize primitive_classes to empty to allow constant_to_value to run.
self.primitive_classes = ()
# object_type is needed to initialize the primitive class values.
self.object_type = self.constant_to_value(object)
# Now fill primitive_classes with the real values using constant_to_value.
self.primitive_classes = {
v: self.constant_to_value(v)
for v in [
int, float, str, unicode, object, types.NoneType, complex,
bool, slice, types.CodeType, types.EllipsisType,
types.ClassType, super
]
}
self.primitive_class_names = [
x.__module__ + "." + x.__name__ for x in self.primitive_classes
]
self.none = abstract.AbstractOrConcreteValue(
None, self.primitive_classes[types.NoneType], self.vm)
self.true = abstract.AbstractOrConcreteValue(
True, self.primitive_classes[bool], self.vm)
self.false = abstract.AbstractOrConcreteValue(
False, self.primitive_classes[bool], self.vm)
self.ellipsis = abstract.AbstractOrConcreteValue(
Ellipsis, self.primitive_classes[types.EllipsisType], self.vm)
self.primitive_class_instances = {}
for name, cls in self.primitive_classes.items():
if name == types.NoneType:
# This is possible because all None instances are the same.
# Without it pytype could not reason that "x is None" is always true, if
# x is indeed None.
instance = self.none
elif name == types.EllipsisType:
instance = self.ellipsis
else:
instance = abstract.Instance(cls, self.vm)
self.primitive_class_instances[name] = instance
self._convert_cache[(abstract.Instance, cls.pytd_cls)] = instance
self.none_type = self.primitive_classes[types.NoneType]
self.oldstyleclass_type = self.primitive_classes[types.ClassType]
self.super_type = self.primitive_classes[super]
self.str_type = self.primitive_classes[str]
self.int_type = self.primitive_classes[int]
self.nothing = abstract.Nothing(self.vm)
self.unsolvable = abstract.Unsolvable(self.vm)
self.empty = abstract.Empty(self.vm)
self.list_type = self.constant_to_value(list)
self.set_type = self.constant_to_value(set)
self.dict_type = self.constant_to_value(dict)
self.type_type = self.constant_to_value(type)
self.module_type = self.constant_to_value(types.ModuleType)
self.function_type = self.constant_to_value(types.FunctionType)
self.tuple_type = self.constant_to_value(tuple)
self.generator_type = self.constant_to_value(types.GeneratorType)
# TODO(dbaum): There isn't a types.IteratorType. This can probably be
# based on typing.Iterator, but that will also require changes to
# convert.py since that assumes all types can be looked up in
# __builtin__.
self.iterator_type = self.constant_to_value(types.ObjectType)
self.bool_values = {
True: self.true,
False: self.false,
None: self.primitive_class_instances[bool],
}
def __init__(self, vm):
self.vm = vm
self.vm.convert = self # to make convert_constant calls below work
self._convert_cache = {}
# Initialize primitive_classes to empty to allow convert_constant to run
self.primitive_classes = ()
# Now fill primitive_classes with the real values using convert_constant
self.primitive_classes = {v: self.convert_constant(v.__name__, v)
for v in [int, long, float, str, unicode, object,
types.NoneType, complex, bool, slice,
types.CodeType, types.EllipsisType,
types.ClassType, super]}
self.none = abstract.AbstractOrConcreteValue(
None, self.primitive_classes[types.NoneType], self.vm,
self.vm.root_cfg_node)
self.true = abstract.AbstractOrConcreteValue(
True, self.primitive_classes[bool], self.vm, self.vm.root_cfg_node)
self.false = abstract.AbstractOrConcreteValue(
False, self.primitive_classes[bool], self.vm, self.vm.root_cfg_node)
self.ellipsis = abstract.AbstractOrConcreteValue(
Ellipsis, self.primitive_classes[types.EllipsisType], self.vm,
self.vm.root_cfg_node)
self.primitive_class_instances = {}
for name, clsvar in self.primitive_classes.items():
instance = abstract.Instance(clsvar, self.vm, self.vm.root_cfg_node)
self.primitive_class_instances[name] = instance
clsval, = clsvar.bindings
self._convert_cache[(abstract.Instance, clsval.data.pytd_cls)] = instance
self.primitive_class_instances[types.NoneType] = self.none
self.none_type = self.primitive_classes[types.NoneType]
self.object_type = self.primitive_classes[object]
self.oldstyleclass_type = self.primitive_classes[types.ClassType]
self.super_type = self.primitive_classes[super]
self.str_type = self.primitive_classes[str]
self.int_type = self.primitive_classes[int]
self.nothing = abstract.Nothing(self.vm)
self.unsolvable = abstract.Unsolvable(self.vm)
self.tuple_type = self.convert_constant("tuple", tuple)
self.list_type = self.convert_constant("list", list)
self.set_type = self.convert_constant("set", set)
self.dict_type = self.convert_constant("dict", dict)
self.type_type = self.convert_constant("type", type)
self.module_type = self.convert_constant("module", types.ModuleType)
self.function_type = self.convert_constant(
"function", types.FunctionType)
self.generator_type = self.convert_constant(
"generator", types.GeneratorType)
# TODO(dbaum): There isn't a types.IteratorType. This can probably be
# based on typing.Iterator, but that will also require changes to
# convert.py since that assumes all types can be looked up in
# __builtin__.
self.iterator_type = self.convert_constant(
"iterator", types.ObjectType)
self.bool_values = {
True: self.true,
False: self.false,
None: self.primitive_class_instances[bool],
}
self.undefined = self.vm.program.NewVariable("undefined")
def construct_constant_from_value(self, name, pyval, subst, node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
name: The name of this constant. Used for naming its attribute variables.
pyval: The python or PyTD value to convert.
subst: The current type parameters.
node: The current CFG node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
"""
if pyval is type:
return abstract.SimpleAbstractValue(name, self.vm)
elif isinstance(pyval, str):
return abstract.AbstractOrConcreteValue(
pyval, self.str_type, self.vm, node)
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(
pyval, self.int_type, self.vm, node)
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm, node)
elif pyval.__class__ in [types.FunctionType,
types.ModuleType,
types.GeneratorType,
type]:
try:
pyclass = self.vm.vmbuiltins.Lookup("__builtin__." + pyval.__name__)
return self.convert_constant_to_value(name, pyclass, subst, node)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.TypeDeclUnit):
data = pyval.constants + pyval.classes + pyval.functions + pyval.aliases
members = {val.name.rsplit(".")[-1]: val
for val in data}
return abstract.Module(self.vm, node, pyval.name, members)
elif isinstance(pyval, pytd.Class):
if "." in pyval.name:
module, base_name = pyval.name.rsplit(".", 1)
cls = abstract.PyTDClass(base_name, pyval, self.vm)
cls.module = module
else:
cls = abstract.PyTDClass(name, pyval, self.vm)
return cls
elif isinstance(pyval, pytd.Function):
signatures = [abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures]
f = abstract.PyTDFunction(
pyval.name, signatures, pyval.kind, self.vm, node)
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.convert_constant_to_value(pyval.name, pyval.cls, subst, node)
elif isinstance(pyval, pytd.NothingType):
return self.nothing
elif isinstance(pyval, pytd.AnythingType):
# TODO(kramm): This should be an Unsolveable. We don't need to solve this.
return self._create_new_unknown_value("AnythingType")
elif isinstance(pyval, pytd.FunctionType):
return self.construct_constant_from_value(
name, pyval.function, subst, node)
elif isinstance(pyval, pytd.UnionType):
return abstract.Union([
self.convert_constant_to_value(pytd.Print(t), t, subst, node)
for t in pyval.type_list], self.vm)
elif isinstance(pyval, pytd.TypeParameter):
return abstract.TypeParameter(pyval.name, self.vm)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.Class):
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in ["__builtin__.type", "__builtin__.property"]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.convert_constant(cls.name, cls, subst, node)
instance = abstract.Instance(mycls, self.vm, node)
log.info("New pytd instance for %s: %r", cls.name, instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.GenericType):
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
instance = abstract.Instance(
self.convert_constant(base_cls.name, base_cls, subst, node),
self.vm, node)
for formal, actual in zip(base_cls.template, cls.parameters):
p = self.convert_constant(
repr(formal), abstract.AsInstance(actual), subst, node)
instance.initialize_type_parameter(node, formal.name, p)
return instance
else:
return self.convert_constant_to_value(name, cls, subst, node)
elif isinstance(pyval, pytd.GenericType):
assert isinstance(pyval.base_type, pytd.ClassType)
type_parameters = utils.LazyDict()
for param, value in zip(pyval.base_type.cls.template, pyval.parameters):
type_parameters.add_lazy_item(
param.name, self.convert_constant_to_value,
param.name, value, subst, node)
cls = self.convert_constant_to_value(
pytd.Print(pyval.base_type), pyval.base_type.cls, subst, node)
return abstract.ParameterizedClass(cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value(self.vm.root_cfg_node,
[self.convert_constant("tuple[%d]" % i, item,
subst, node)
for i, item in enumerate(pyval)])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def new_tuple(*args):
pyval = tuple(maybe_var(a) for a in args)
return abstract.AbstractOrConcreteValue(
pyval, self._vm.convert.tuple_type, self._vm, self._node)
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if isinstance(pyval, str):
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, long):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
if pyval is types.FunctionType:
classname = "typing.Callable"
else:
classname = "__builtin__." + pyval.__name__
try:
return self.name_to_value(classname, subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.TypeDeclUnit):
data = (pyval.constants + pyval.type_params + pyval.classes +
pyval.functions + pyval.aliases)
members = {val.name.rsplit(".")[-1]: val for val in data}
return abstract.Module(self.vm, pyval.name, members, pyval)
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.object":
return self.object_type
elif isinstance(pyval,
pytd.Class) and pyval.name == "types.ModuleType":
return self.module_type
elif isinstance(pyval, pytd.Class):
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.ExternalFunction):
module, _, name = pyval.name.partition(".")
assert module == "__builtin__", "PYTHONCODE allowed only in __builtin__"
return abstract.merge_values(
self.vm.frame.f_globals.members[name].data, self.vm)
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.nothing
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.Class):
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
instance.make_template_unsolvable(
cls.template, self.vm.root_cfg_node)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.GenericType):
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
assert len(cls.parameters) <= len(base_cls.template)
for formal, actual in zip(base_cls.template,
cls.parameters):
p = self.constant_to_var(abstract.AsInstance(actual),
subst, self.vm.root_cfg_node)
instance.initialize_type_parameter(
get_node(), formal.name, p)
return instance
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
assert isinstance(pyval.base_type, pytd.ClassType)
base_cls = self.constant_to_value(pyval.base_type.cls, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = range(len(pyval.parameters)) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = range(len(
pyval.args)) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in pyval.base_type.cls.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = utils.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def __init__(self, vm):
self.vm = vm
self.vm.convert = self # to make constant_to_var calls below work
self.pytd_convert = output.Converter()
self._convert_cache = {}
# Initialize primitive_classes to empty to allow constant_to_var to run
self.primitive_classes = ()
# Now fill primitive_classes with the real values using constant_to_var
self.primitive_classes = {
v: self.constant_to_var(v.__name__, v)
for v in [
int, float, str, unicode, object, types.NoneType, complex,
bool, slice, types.CodeType, types.EllipsisType,
types.ClassType, super
]
}
self.none = abstract.AbstractOrConcreteValue(
None, self.primitive_classes[types.NoneType], self.vm,
self.vm.root_cfg_node)
self.true = abstract.AbstractOrConcreteValue(
True, self.primitive_classes[bool], self.vm, self.vm.root_cfg_node)
self.false = abstract.AbstractOrConcreteValue(
False, self.primitive_classes[bool], self.vm,
self.vm.root_cfg_node)
self.ellipsis = abstract.AbstractOrConcreteValue(
Ellipsis, self.primitive_classes[types.EllipsisType], self.vm,
self.vm.root_cfg_node)
self.primitive_class_instances = {}
for name, clsvar in self.primitive_classes.items():
if name == types.NoneType:
# This is possible because all None instances are the same.
# Without it pytype could not reason that "x is None" is always true, if
# x is indeed None.
instance = self.none
elif name == types.EllipsisType:
instance = self.ellipsis
else:
instance = abstract.Instance(clsvar, self.vm,
self.vm.root_cfg_node)
self.primitive_class_instances[name] = instance
clsval, = clsvar.bindings
self._convert_cache[(abstract.Instance,
clsval.data.pytd_cls)] = instance
self.none_type = self.primitive_classes[types.NoneType]
self.object_type = self.primitive_classes[object]
self.oldstyleclass_type = self.primitive_classes[types.ClassType]
self.super_type = self.primitive_classes[super]
self.str_type = self.primitive_classes[str]
self.int_type = self.primitive_classes[int]
self.nothing = abstract.Nothing(self.vm)
self.unsolvable = abstract.Unsolvable(self.vm)
self.empty = abstract.Empty(self.vm)
self.tuple_type = self.constant_to_var("tuple", tuple)
self.list_type = self.constant_to_var("list", list)
self.set_type = self.constant_to_var("set", set)
self.frozenset_type = self.constant_to_var("frozenset", frozenset)
self.dict_type = self.constant_to_var("dict", dict)
self.type_type = self.constant_to_var("type", type)
self.module_type = self.constant_to_var("module", types.ModuleType)
self.function_type = self.constant_to_var("function",
types.FunctionType)
self.generator_type = self.constant_to_var("generator",
types.GeneratorType)
# TODO(dbaum): There isn't a types.IteratorType. This can probably be
# based on typing.Iterator, but that will also require changes to
# convert.py since that assumes all types can be looked up in
# __builtin__.
self.iterator_type = self.constant_to_var("iterator", types.ObjectType)
self.bool_values = {
True: self.true,
False: self.false,
None: self.primitive_class_instances[bool],
}
self.empty_type = self.empty.to_variable(self.vm.root_cfg_node)
object_val, = self.object_type.data
object_val.load_lazy_attribute("__new__")
self.object_new, = object_val.members["__new__"].data
self.typing_overlay = typing.TypingOverlay(self.vm,
self.vm.root_cfg_node)
