def _get_attribute(self, node, obj, cls, name, valself): """Get an attribute from an object or its class. The underlying method called by all of the (_)get_(x_)attribute methods. Attempts to resolve an attribute first with __getattribute__, then by fetching it from the object, then by fetching it from the class, and finally with __getattr__. Arguments: node: The current node. obj: The object. cls: The object's class, may be None. name: The attribute name. valself: The binding to the self reference. Returns: A tuple of the node and the attribute, or None if it was not found. """ if cls: # A __getattribute__ on the class controls all attribute access. node, attr = self._get_attribute_computed( node, cls, name, valself, compute_function="__getattribute__") else: attr = None if attr is None: # Check for the attribute on the instance. if isinstance(obj, class_mixin.Class): # A class is an instance of its metaclass. node, attr = self._lookup_from_mro_and_handle_descriptors( node, obj, name, valself, skip=()) else: node, attr = self._get_member(node, obj, name) if attr is None and cls: # Check for the attribute on the class. node, attr = self.get_attribute(node, cls, name, valself) if attr is None: # Fall back to __getattr__ if the attribute doesn't otherwise exist. node, attr = self._get_attribute_computed( node, cls, name, valself, compute_function="__getattr__") if attr is None: for base in obj.mro: if not isinstance(base, abstract.InterpreterClass): break annots = abstract_utils.get_annotations_dict(base.members) if annots: typ = annots.get_type(node, name) if typ: # An attribute has been declared but not defined, e.g., # class Foo: # bar: int _, attr = self.vm.annotations_util.init_annotation(node, name, typ) break if attr is not None: attr = self._filter_var(node, attr) if attr is None and obj.maybe_missing_members: # The VM hit maximum depth while initializing this instance, so it may # have attributes that we don't know about. attr = self.vm.new_unsolvable(node) return node, attr
def _handle_initvar(self, node, cls, name, value, orig): """Unpack or delete an initvar in the class annotations.""" initvar = match_initvar(value) if not initvar: return None annots = abstract_utils.get_annotations_dict(cls.members) if orig is None: # InitVars without a default do not get retained. del annots[name] else: annots[name] = initvar.to_variable(node) return initvar
def _handle_initvar(self, node, cls, name, value, orig): """Unpack or delete an initvar in the class annotations.""" initvar = match_initvar(value) if not initvar: return None annots = abstract_utils.get_annotations_dict(cls.members) # The InitVar annotation is not retained as a class member, but any default # value is retained. del annots[name] value = initvar.instantiate(node) if orig is not None: cls.members[name] = value return value
def pytd_for_types(self, defs): # If a variable is annotated, we'll always output that type. annotated_names = set() data = [] pytd_convert = self.convert.pytd_convert annots = abstract_utils.get_annotations_dict(defs) for name, t in pytd_convert.annotations_to_instance_types( self.exitpoint, annots): annotated_names.add(name) data.append(pytd.Constant(name, t)) for name, var in defs.items(): if (name in output.TOP_LEVEL_IGNORE or name in annotated_names or self._is_typing_member(name, var)): continue options = var.FilteredData(self.exitpoint, strict=False) if (len(options) > 1 and not all(isinstance(o, abstract.FUNCTION_TYPES) for o in options)): if all(isinstance(o, (abstract.ParameterizedClass, abstract.TypeParameter, abstract.Union)) for o in options ) and self.options.preserve_union_macros: # type alias data.append(pytd_utils.JoinTypes(t.to_pytd_def(self.exitpoint, name) for t in options)) else: # It's ambiguous whether this is a type, a function or something # else, so encode it as a constant. combined_types = pytd_utils.JoinTypes(t.to_type(self.exitpoint) for t in options) data.append(pytd.Constant(name, combined_types)) elif options: for option in options: try: d = option.to_pytd_def(self.exitpoint, name) # Deep definition except NotImplementedError: d = option.to_type(self.exitpoint) # Type only if isinstance(d, pytd.NothingType): if isinstance(option, abstract.Empty): d = pytd.AnythingType() else: assert isinstance(option, typing_overlay.NoReturn) if isinstance(d, pytd.Type) and not isinstance(d, pytd.TypeParameter): data.append(pytd.Constant(name, d)) else: data.append(d) else: log.error("No visible options for %s", name) data.append(pytd.Constant(name, pytd.AnythingType())) return pytd_utils.WrapTypeDeclUnit("inferred", data)
def pytd_for_types(self, defs): data = [] pytd_convert = self.convert.pytd_convert annots = abstract_utils.get_annotations_dict(defs) or {} for name, t in pytd_convert.uninitialized_annotations_to_instance_types( self.exitpoint, annots, defs): data.append(pytd.Constant(name, t)) for name, var in defs.items(): if name in output.TOP_LEVEL_IGNORE or self._is_builtin( name, var.data): continue options = [] for value, is_annotation in pytd_convert.get_annotated_values( self.exitpoint, name, var, annots): if is_annotation: data.append(pytd.Constant(name, value)) else: options.append(value) if (len(options) > 1 and not all( isinstance(o, abstract.FUNCTION_TYPES) for o in options)): # It's ambiguous whether this is a type, a function or something # else, so encode it as a constant. combined_types = pytd_utils.JoinTypes( t.to_type(self.exitpoint) for t in options) data.append(pytd.Constant(name, combined_types)) elif options: for option in options: try: d = option.to_pytd_def(self.exitpoint, name) # Deep definition except NotImplementedError: d = option.to_type(self.exitpoint) # Type only if isinstance(d, pytd.NothingType): if isinstance(option, abstract.Empty): d = pytd.AnythingType() else: assert isinstance(option, typing_overlay.NoReturn) if isinstance(d, pytd.Type) and not isinstance( d, pytd.TypeParameter): data.append(pytd.Constant(name, d)) else: data.append(d) else: log.error("No visible options for %s", name) data.append(pytd.Constant(name, pytd.AnythingType())) return pytd_utils.WrapTypeDeclUnit("inferred", data)
def _class_to_def(self, node, v, class_name): """Convert an InterpreterClass to a PyTD definition.""" methods = {} constants = collections.defaultdict(pytd_utils.TypeBuilder) annots = abstract_utils.get_annotations_dict(v.members) or {} for name, t in self.uninitialized_annotations_to_instance_types( node, annots, v.members): constants[name].add_type(t) # class-level attributes for name, member in v.members.items(): if name in CLASS_LEVEL_IGNORE: continue for value in member.FilteredData(self.vm.exitpoint, strict=False): if isinstance(value, special_builtins.PropertyInstance): # For simplicity, output properties as constants, since our parser # turns them into constants anyway. if value.fget: for typ in self._function_to_return_types( node, value.fget): constants[name].add_type(typ) else: constants[name].add_type(pytd.AnythingType()) elif isinstance(value, special_builtins.StaticMethodInstance): try: methods[name] = self._static_method_to_def( node, value, name, pytd.STATICMETHOD) except abstract_utils.ConversionError: constants[name].add_type(pytd.AnythingType()) elif isinstance(value, special_builtins.ClassMethodInstance): try: methods[name] = self._class_method_to_def( node, value, name, pytd.CLASSMETHOD) except abstract_utils.ConversionError: constants[name].add_type(pytd.AnythingType()) elif isinstance(value, abstract.Function): # TODO(rechen): Removing mutations altogether won't work for generic # classes. To support those, we'll need to change the mutated type's # base to the current class, rename aliased type parameters, and # replace any parameter not in the class or function template with # its upper value. methods[name] = self.value_to_pytd_def( node, value, name).Visit(visitors.DropMutableParameters()) else: cls = self.vm.convert.merge_classes([value]) node, attr = self.vm.attribute_handler.get_attribute( node, cls, "__get__") if attr: # This attribute is a descriptor. Its type is the return value of # its __get__ method. for typ in self._function_to_return_types(node, attr): constants[name].add_type(typ) else: constants[name].add_type(value.to_type(node)) # instance-level attributes for instance in set(v.instances): for name, member in instance.members.items(): if name not in CLASS_LEVEL_IGNORE: for value in member.FilteredData(self.vm.exitpoint, strict=False): constants[name].add_type(value.to_type(node)) for name in list(methods): if name in constants: # If something is both a constant and a method, it means that the class # is, at some point, overwriting its own methods with an attribute. del methods[name] constants[name].add_type(pytd.AnythingType()) constants = [ pytd.Constant(name, builder.build()) for name, builder in constants.items() if builder ] metaclass = v.metaclass(node) if metaclass is not None: metaclass = metaclass.get_instance_type(node) # Some of the class's bases may not be in global scope, so they won't show # up in the output. In that case, fold the base class's type information # into this class's pytd. bases = [] missing_bases = [] for basevar in v.bases(): if basevar.data == [self.vm.convert.oldstyleclass_type]: continue elif len(basevar.bindings) == 1: b, = basevar.data if b.official_name is None and isinstance( b, abstract.InterpreterClass): missing_bases.append(b) else: bases.append(b.get_instance_type(node)) else: bases.append( pytd_utils.JoinTypes( b.get_instance_type(node) for b in basevar.data)) # Collect nested classes # TODO(mdemello): We cannot put these in the output yet; they fail in # load_dependencies because of the dotted class name (google/pytype#150) classes = [ self._class_to_def(node, x, x.name) for x in v.get_inner_classes() ] classes = [x.Replace(name=class_name + "." + x.name) for x in classes] cls = pytd.Class(name=class_name, metaclass=metaclass, parents=tuple(bases), methods=tuple(methods.values()), constants=tuple(constants), classes=(), slots=v.slots, template=()) for base in missing_bases: base_cls = self.value_to_pytd_def(node, base, base.name) cls = pytd_utils.MergeBaseClass(cls, base_cls) return cls
def _class_to_def(self, node, v, class_name): """Convert an InterpreterClass to a PyTD definition.""" self._scopes.append(class_name) methods = {} constants = collections.defaultdict(pytd_utils.TypeBuilder) annots = abstract_utils.get_annotations_dict(v.members) annotated_names = set() def add_constants(iterator): for name, t in iterator: if t is None: # Remove the entry from constants annotated_names.add(name) elif name not in annotated_names: constants[name].add_type(t) annotated_names.add(name) add_constants( self._ordered_attrs_to_instance_types(node, v.metadata, annots)) add_constants(self.annotations_to_instance_types(node, annots)) def get_decorated_method(name, value, func_slot): fvar = getattr(value, func_slot) func = abstract_utils.get_atomic_value(fvar, abstract.Function) defn = self.value_to_pytd_def(node, func, name) defn = defn.Visit(visitors.DropMutableParameters()) return defn def add_decorated_method(name, value, kind): try: defn = get_decorated_method(name, value, "func") except (AttributeError, abstract_utils.ConversionError): constants[name].add_type(pytd.AnythingType()) return defn = defn.Replace(kind=kind) methods[name] = defn # If decorators are output as aliases to NamedTypes, they will be converted # to Functions and fail a verification step if those functions have type # parameters. Since we just want the function name, and since we have a # fully resolved name at this stage, we just output a minimal pytd.Function sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ()) decorators = [ pytd.Alias(x, pytd.Function(x, (sig, ), pytd.MethodTypes.METHOD, 0)) for x in v.decorators ] # class-level attributes for name, member in v.members.items(): if name in CLASS_LEVEL_IGNORE or name in annotated_names: continue for value in member.FilteredData(self.vm.exitpoint, strict=False): if isinstance(value, special_builtins.PropertyInstance): # For simplicity, output properties as constants, since our parser # turns them into constants anyway. if value.fget: for typ in self._function_to_return_types( node, value.fget): constants[name].add_type( pytd.Annotated(typ, ("'property'", ))) else: constants[name].add_type( pytd.Annotated(pytd.AnythingType(), ("'property'", ))) elif isinstance(value, special_builtins.StaticMethodInstance): add_decorated_method(name, value, pytd.MethodTypes.STATICMETHOD) elif isinstance(value, special_builtins.ClassMethodInstance): add_decorated_method(name, value, pytd.MethodTypes.CLASSMETHOD) elif isinstance(value, abstract.Function): # value_to_pytd_def returns different pytd node types depending on the # input type, which pytype struggles to reason about. method = cast(pytd.Function, self.value_to_pytd_def(node, value, name)) keep = lambda name: not name or name.startswith(v.name) signatures = tuple( s for s in method.signatures if not s.params or keep(s.params[0].type.name)) if signatures and signatures != method.signatures: # Filter out calls made from subclasses unless they are the only # ones recorded; when inferring types for ParentClass.__init__, we # do not want `self: Union[ParentClass, Subclass]`. method = method.Replace(signatures=signatures) # TODO(rechen): Removing mutations altogether won't work for generic # classes. To support those, we'll need to change the mutated type's # base to the current class, rename aliased type parameters, and # replace any parameter not in the class or function template with # its upper value. methods[name] = method.Visit( visitors.DropMutableParameters()) else: cls = self.vm.convert.merge_classes([value]) node, attr = self.vm.attribute_handler.get_attribute( node, cls, "__get__") if attr: # This attribute is a descriptor. Its type is the return value of # its __get__ method. for typ in self._function_to_return_types(node, attr): constants[name].add_type(typ) else: constants[name].add_type(value.to_type(node)) # Instance-level attributes: all attributes from 'canonical' instances (that # is, ones created by analyze.py:analyze_class()) are added. Attributes from # non-canonical instances are added if their canonical values do not contain # type parameters. ignore = set(annotated_names) canonical_attributes = set() def add_attributes_from(instance): for name, member in instance.members.items(): if name in CLASS_LEVEL_IGNORE or name in ignore: continue for value in member.FilteredData(self.vm.exitpoint, strict=False): typ = value.to_type(node) if pytd_utils.GetTypeParameters(typ): # This attribute's type comes from an annotation that contains a # type parameter; we do not want to merge in substituted values of # the type parameter. canonical_attributes.add(name) constants[name].add_type(typ) for instance in v.canonical_instances: add_attributes_from(instance) ignore |= canonical_attributes for instance in v.instances - v.canonical_instances: add_attributes_from(instance) for name in list(methods): if name in constants: # If something is both a constant and a method, it means that the class # is, at some point, overwriting its own methods with an attribute. del methods[name] constants[name].add_type(pytd.AnythingType()) constants = [ pytd.Constant(name, builder.build()) for name, builder in constants.items() if builder ] metaclass = v.metaclass(node) if metaclass is not None: metaclass = metaclass.get_instance_type(node) # Some of the class's bases may not be in global scope, so they won't show # up in the output. In that case, fold the base class's type information # into this class's pytd. bases = [] missing_bases = [] for basevar in v.bases(): if basevar.data == [self.vm.convert.oldstyleclass_type]: continue elif len(basevar.bindings) == 1: b, = basevar.data if b.official_name is None and isinstance( b, abstract.InterpreterClass): missing_bases.append(b) else: bases.append(b.get_instance_type(node)) else: bases.append( pytd_utils.JoinTypes( b.get_instance_type(node) for b in basevar.data)) # Collect nested classes # TODO(mdemello): We cannot put these in the output yet; they fail in # load_dependencies because of the dotted class name (google/pytype#150) classes = [ self._class_to_def(node, x, x.name) for x in v.get_inner_classes() ] classes = [x.Replace(name=class_name + "." + x.name) for x in classes] cls = pytd.Class(name=class_name, metaclass=metaclass, parents=tuple(bases), methods=tuple(methods.values()), constants=tuple(constants), classes=(), decorators=tuple(decorators), slots=v.slots, template=()) for base in missing_bases: base_cls = self.value_to_pytd_def(node, base, base.name) cls = pytd_utils.MergeBaseClass(cls, base_cls) self._scopes.pop() return cls