def _get_attribute(self, node, obj, cls, name, valself):
"""Get an attribute from an object or its class.
The underlying method called by all of the (_)get_(x_)attribute methods.
Attempts to resolve an attribute first with __getattribute__, then by
fetching it from the object, then by fetching it from the class, and
finally with __getattr__.
Arguments:
node: The current node.
obj: The object.
cls: The object's class, may be None.
name: The attribute name.
valself: The binding to the self reference.
Returns:
A tuple of the node and the attribute, or None if it was not found.
"""
if cls:
# A __getattribute__ on the class controls all attribute access.
node, attr = self._get_attribute_computed(
node, cls, name, valself, compute_function="__getattribute__")
else:
attr = None
if attr is None:
# Check for the attribute on the instance.
if isinstance(obj, class_mixin.Class):
# A class is an instance of its metaclass.
node, attr = self._lookup_from_mro_and_handle_descriptors(
node, obj, name, valself, skip=())
else:
node, attr = self._get_member(node, obj, name)
if attr is None and cls:
# Check for the attribute on the class.
node, attr = self.get_attribute(node, cls, name, valself)
if attr is None:
# Fall back to __getattr__ if the attribute doesn't otherwise exist.
node, attr = self._get_attribute_computed(
node, cls, name, valself, compute_function="__getattr__")
if attr is None:
for base in obj.mro:
if not isinstance(base, abstract.InterpreterClass):
break
annots = abstract_utils.get_annotations_dict(base.members)
if annots:
typ = annots.get_type(node, name)
if typ:
# An attribute has been declared but not defined, e.g.,
# class Foo:
# bar: int
_, attr = self.vm.annotations_util.init_annotation(node, name, typ)
break
if attr is not None:
attr = self._filter_var(node, attr)
if attr is None and obj.maybe_missing_members:
# The VM hit maximum depth while initializing this instance, so it may
# have attributes that we don't know about.
attr = self.vm.new_unsolvable(node)
return node, attr
def _handle_initvar(self, node, cls, name, value, orig):
"""Unpack or delete an initvar in the class annotations."""
initvar = match_initvar(value)
if not initvar:
return None
annots = abstract_utils.get_annotations_dict(cls.members)
if orig is None:
# InitVars without a default do not get retained.
del annots[name]
else:
annots[name] = initvar.to_variable(node)
return initvar
def _handle_initvar(self, node, cls, name, value, orig):
"""Unpack or delete an initvar in the class annotations."""
initvar = match_initvar(value)
if not initvar:
return None
annots = abstract_utils.get_annotations_dict(cls.members)
# The InitVar annotation is not retained as a class member, but any default
# value is retained.
del annots[name]
value = initvar.instantiate(node)
if orig is not None:
cls.members[name] = value
return value
def pytd_for_types(self, defs):
# If a variable is annotated, we'll always output that type.
annotated_names = set()
data = []
pytd_convert = self.convert.pytd_convert
annots = abstract_utils.get_annotations_dict(defs)
for name, t in pytd_convert.annotations_to_instance_types(
self.exitpoint, annots):
annotated_names.add(name)
data.append(pytd.Constant(name, t))
for name, var in defs.items():
if (name in output.TOP_LEVEL_IGNORE or name in annotated_names or
self._is_typing_member(name, var)):
continue
options = var.FilteredData(self.exitpoint, strict=False)
if (len(options) > 1 and
not all(isinstance(o, abstract.FUNCTION_TYPES) for o in options)):
if all(isinstance(o, (abstract.ParameterizedClass,
abstract.TypeParameter,
abstract.Union)) for o in options
) and self.options.preserve_union_macros: # type alias
data.append(pytd_utils.JoinTypes(t.to_pytd_def(self.exitpoint, name)
for t in options))
else:
# It's ambiguous whether this is a type, a function or something
# else, so encode it as a constant.
combined_types = pytd_utils.JoinTypes(t.to_type(self.exitpoint)
for t in options)
data.append(pytd.Constant(name, combined_types))
elif options:
for option in options:
try:
d = option.to_pytd_def(self.exitpoint, name) # Deep definition
except NotImplementedError:
d = option.to_type(self.exitpoint) # Type only
if isinstance(d, pytd.NothingType):
if isinstance(option, abstract.Empty):
d = pytd.AnythingType()
else:
assert isinstance(option, typing_overlay.NoReturn)
if isinstance(d, pytd.Type) and not isinstance(d, pytd.TypeParameter):
data.append(pytd.Constant(name, d))
else:
data.append(d)
else:
log.error("No visible options for %s", name)
data.append(pytd.Constant(name, pytd.AnythingType()))
return pytd_utils.WrapTypeDeclUnit("inferred", data)
def pytd_for_types(self, defs):
data = []
pytd_convert = self.convert.pytd_convert
annots = abstract_utils.get_annotations_dict(defs) or {}
for name, t in pytd_convert.uninitialized_annotations_to_instance_types(
self.exitpoint, annots, defs):
data.append(pytd.Constant(name, t))
for name, var in defs.items():
if name in output.TOP_LEVEL_IGNORE or self._is_builtin(
name, var.data):
continue
options = []
for value, is_annotation in pytd_convert.get_annotated_values(
self.exitpoint, name, var, annots):
if is_annotation:
data.append(pytd.Constant(name, value))
else:
options.append(value)
if (len(options) > 1 and not all(
isinstance(o, abstract.FUNCTION_TYPES) for o in options)):
# It's ambiguous whether this is a type, a function or something
# else, so encode it as a constant.
combined_types = pytd_utils.JoinTypes(
t.to_type(self.exitpoint) for t in options)
data.append(pytd.Constant(name, combined_types))
elif options:
for option in options:
try:
d = option.to_pytd_def(self.exitpoint,
name) # Deep definition
except NotImplementedError:
d = option.to_type(self.exitpoint) # Type only
if isinstance(d, pytd.NothingType):
if isinstance(option, abstract.Empty):
d = pytd.AnythingType()
else:
assert isinstance(option,
typing_overlay.NoReturn)
if isinstance(d, pytd.Type) and not isinstance(
d, pytd.TypeParameter):
data.append(pytd.Constant(name, d))
else:
data.append(d)
else:
log.error("No visible options for %s", name)
data.append(pytd.Constant(name, pytd.AnythingType()))
return pytd_utils.WrapTypeDeclUnit("inferred", data)
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members) or {}
for name, t in self.uninitialized_annotations_to_instance_types(
node, annots, v.members):
constants[name].add_type(t)
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE:
continue
for value in member.FilteredData(self.vm.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(typ)
else:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.StaticMethodInstance):
try:
methods[name] = self._static_method_to_def(
node, value, name, pytd.STATICMETHOD)
except abstract_utils.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.ClassMethodInstance):
try:
methods[name] = self._class_method_to_def(
node, value, name, pytd.CLASSMETHOD)
except abstract_utils.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, abstract.Function):
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = self.value_to_pytd_def(
node, value,
name).Visit(visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes([value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in set(v.instances):
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(self.vm.exitpoint,
strict=False):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
return cls
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
self._scopes.append(class_name)
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members)
annotated_names = set()
def add_constants(iterator):
for name, t in iterator:
if t is None:
# Remove the entry from constants
annotated_names.add(name)
elif name not in annotated_names:
constants[name].add_type(t)
annotated_names.add(name)
add_constants(
self._ordered_attrs_to_instance_types(node, v.metadata, annots))
add_constants(self.annotations_to_instance_types(node, annots))
def get_decorated_method(name, value, func_slot):
fvar = getattr(value, func_slot)
func = abstract_utils.get_atomic_value(fvar, abstract.Function)
defn = self.value_to_pytd_def(node, func, name)
defn = defn.Visit(visitors.DropMutableParameters())
return defn
def add_decorated_method(name, value, kind):
try:
defn = get_decorated_method(name, value, "func")
except (AttributeError, abstract_utils.ConversionError):
constants[name].add_type(pytd.AnythingType())
return
defn = defn.Replace(kind=kind)
methods[name] = defn
# If decorators are output as aliases to NamedTypes, they will be converted
# to Functions and fail a verification step if those functions have type
# parameters. Since we just want the function name, and since we have a
# fully resolved name at this stage, we just output a minimal pytd.Function
sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ())
decorators = [
pytd.Alias(x, pytd.Function(x, (sig, ), pytd.MethodTypes.METHOD,
0)) for x in v.decorators
]
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE or name in annotated_names:
continue
for value in member.FilteredData(self.vm.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(
pytd.Annotated(typ, ("'property'", )))
else:
constants[name].add_type(
pytd.Annotated(pytd.AnythingType(),
("'property'", )))
elif isinstance(value, special_builtins.StaticMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.STATICMETHOD)
elif isinstance(value, special_builtins.ClassMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.CLASSMETHOD)
elif isinstance(value, abstract.Function):
# value_to_pytd_def returns different pytd node types depending on the
# input type, which pytype struggles to reason about.
method = cast(pytd.Function,
self.value_to_pytd_def(node, value, name))
keep = lambda name: not name or name.startswith(v.name)
signatures = tuple(
s for s in method.signatures
if not s.params or keep(s.params[0].type.name))
if signatures and signatures != method.signatures:
# Filter out calls made from subclasses unless they are the only
# ones recorded; when inferring types for ParentClass.__init__, we
# do not want `self: Union[ParentClass, Subclass]`.
method = method.Replace(signatures=signatures)
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = method.Visit(
visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes([value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# Instance-level attributes: all attributes from 'canonical' instances (that
# is, ones created by analyze.py:analyze_class()) are added. Attributes from
# non-canonical instances are added if their canonical values do not contain
# type parameters.
ignore = set(annotated_names)
canonical_attributes = set()
def add_attributes_from(instance):
for name, member in instance.members.items():
if name in CLASS_LEVEL_IGNORE or name in ignore:
continue
for value in member.FilteredData(self.vm.exitpoint,
strict=False):
typ = value.to_type(node)
if pytd_utils.GetTypeParameters(typ):
# This attribute's type comes from an annotation that contains a
# type parameter; we do not want to merge in substituted values of
# the type parameter.
canonical_attributes.add(name)
constants[name].add_type(typ)
for instance in v.canonical_instances:
add_attributes_from(instance)
ignore |= canonical_attributes
for instance in v.instances - v.canonical_instances:
add_attributes_from(instance)
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
decorators=tuple(decorators),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
self._scopes.pop()
return cls
