def testFlattenSuperclasses(self):
cls_a = pytd.Class("A", None, (), (), (), ())
cls_b = pytd.Class("B", None, (cls_a,), (), (), ())
cls_c = pytd.Class("C", None, (cls_a,), (), (), ())
cls_d = pytd.Class("D", None, (cls_c,), (), (), ())
cls_e = pytd.Class("E", None, (cls_d, cls_b), (), (), ())
self.assertItemsEqual(mro.flattened_superclasses(cls_e),
[cls_a, cls_b, cls_c, cls_d, cls_e])
def new_named_tuple(self, base_name, fields):
"""Return a type for a named tuple (implicitly generates a class).
Args:
base_name: The named tuple's name.
fields: A list of (name, type) tuples.
Returns:
A NamedType() for the generated class that describes the named tuple.
"""
# Handle previously defined NamedTuples with the same name
prev_list = self._generated_classes[base_name]
name_dedup = "~%d" % len(prev_list) if prev_list else ""
class_name = "`%s%s`" % (base_name, name_dedup)
class_parent = self._heterogeneous_tuple(pytd.NamedType("tuple"),
tuple(t for _, t in fields))
class_constants = tuple(pytd.Constant(n, t) for n, t in fields)
nt_class = pytd.Class(name=class_name,
metaclass=None,
parents=(class_parent, ),
methods=(),
constants=class_constants,
template=())
self._generated_classes[base_name].append(nt_class)
return pytd.NamedType(nt_class.name)
def pytd_classes_for_call_traces(self):
class_to_records = collections.defaultdict(list)
for call_record in self._method_calls:
args = call_record.positional_arguments
if not any(isinstance(a.data, abstract.Unknown) for a in args):
# We don't need to record call signatures that don't involve
# unknowns - there's nothing to solve for.
continue
cls = args[0].data.get_class()
if isinstance(cls, abstract.PyTDClass):
class_to_records[cls].append(call_record)
classes = []
for cls, call_records in class_to_records.items():
full_name = cls.module + "." + cls.name if cls.module else cls.name
classes.append(
pytd.Class(
name=self._pack_name(full_name),
metaclass=None,
parents=(pytd.NamedType("__builtin__.object"),
), # not used in solver
methods=tuple(self._call_traces_to_function(call_records)),
constants=(),
classes=(),
slots=None,
template=(),
))
return classes
def MergeBaseClass(cls, base):
"""Merge a base class into a subclass.
Arguments:
cls: The subclass to merge values into. pytd.Class.
base: The superclass whose values will be merged. pytd.Class.
Returns:
a pytd.Class of the two merged classes.
"""
bases = tuple(b for b in cls.parents if b != base)
bases += tuple(b for b in base.parents if b not in bases)
method_names = [m.name for m in cls.methods]
methods = cls.methods + tuple(
m for m in base.methods if m.name not in method_names)
constant_names = [c.name for c in cls.constants]
constants = cls.constants + tuple(
c for c in base.constants if c.name not in constant_names)
class_names = [c.name for c in cls.classes]
classes = cls.classes + tuple(
c for c in base.classes if c.name not in class_names)
if cls.slots:
slots = cls.clots + tuple(
s for s in base.slots or () if s not in cls.slots)
else:
slots = base.slots
return pytd.Class(name=cls.name,
metaclass=cls.metaclass or base.metaclass,
parents=bases,
methods=methods,
constants=constants,
classes=classes,
slots=slots,
template=cls.template or base.template)
def new_named_tuple(self, base_name, fields):
"""Return a type for a named tuple (implicitly generates a class).
Args:
base_name: The named tuple's name.
fields: A list of (name, type) tuples.
Returns:
A NamedType() for the generated class that describes the named tuple.
"""
# Handle previously defined NamedTuples with the same name
prev_list = self._generated_classes[base_name]
class_name = "namedtuple-%s-%d" % (base_name, len(prev_list))
class_parent = self._heterogeneous_tuple(pytd.NamedType("tuple"),
tuple(t for _, t in fields))
class_constants = tuple(pytd.Constant(n, t) for n, t in fields)
# Since the user-defined fields are the only namedtuple attributes commonly
# used, we define all the other attributes as Any for simplicity.
class_constants += tuple(pytd.Constant(name, pytd.AnythingType())
for name in self._NAMEDTUPLE_MEMBERS)
methods = _merge_method_signatures(
[self._namedtuple_new(class_name, fields), self._namedtuple_init()])
nt_class = pytd.Class(name=class_name,
metaclass=None,
parents=(class_parent,),
methods=tuple(methods),
constants=class_constants,
slots=tuple(n for n, _ in fields),
template=())
self._generated_classes[base_name].append(nt_class)
return pytd.NamedType(nt_class.name)
def to_structural_def(self, node, class_name):
"""Convert this Unknown to a pytd.Class."""
self_param = (pytd.Parameter("self", pytd.AnythingType(),
pytd.ParameterKind.REGULAR, False,
None), )
starargs = None
starstarargs = None
def _make_sig(args, ret):
return pytd.Signature(self_param + self._make_params(node, args),
starargs,
starstarargs,
return_type=Unknown._to_pytd(node, ret),
exceptions=(),
template=())
calls = tuple(
pytd_utils.OrderedSet(
_make_sig(args, ret) for args, _, ret in self._calls))
if calls:
methods = (pytd.Function("__call__", calls,
pytd.MethodKind.METHOD), )
else:
methods = ()
return pytd.Class(name=class_name,
metaclass=None,
bases=(pytd.NamedType("builtins.object"), ),
methods=methods,
constants=tuple(
pytd.Constant(name, Unknown._to_pytd(node, c))
for name, c in self.members.items()),
classes=(),
decorators=(),
slots=None,
template=())
def _new_named_tuple(
self,
class_name: str,
fields: List[Tuple[str, Any]]
) -> pytd.Class:
"""Generates a pytd class for a named tuple.
Args:
class_name: The name of the generated class
fields: A list of (name, type) tuples.
Returns:
A generated class that describes the named tuple.
"""
class_parent = types.heterogeneous_tuple(pytd.NamedType("tuple"),
tuple(t for _, t in fields))
class_constants = tuple(pytd.Constant(n, t) for n, t in fields)
# Since the user-defined fields are the only namedtuple attributes commonly
# used, we define all the other attributes as Any for simplicity.
class_constants += tuple(pytd.Constant(name, pytd.AnythingType())
for name in _NAMEDTUPLE_MEMBERS)
methods = function.merge_method_signatures(
[self._make_new(class_name, fields), self._make_init()])
return pytd.Class(name=class_name,
metaclass=None,
parents=(class_parent,),
methods=tuple(methods),
constants=class_constants,
decorators=(),
classes=(),
slots=tuple(n for n, _ in fields),
template=())
def VisitClass(self, node):
return pytd.Class(name=node.name,
metaclass=node.metaclass,
parents=node.parents,
methods=tuple(sorted(node.methods)),
constants=tuple(sorted(node.constants)),
classes=tuple(sorted(node.classes)),
slots=tuple(sorted(node.slots))
if node.slots is not None else None,
template=node.template)
def testVerifyHeterogeneousTuple(self):
# Error: does not inherit from Generic
base = pytd.ClassType("tuple")
base.cls = pytd.Class("tuple", None, (), (), (), (), None, ())
t1 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float")))
self.assertRaises(visitors.ContainerError,
lambda: t1.Visit(visitors.VerifyContainers()))
# Error: Generic[str, float]
gen = pytd.ClassType("typing.Generic")
gen.cls = pytd.Class("typing.Generic", None, (), (), (), (), None, ())
t2 = pytd.TupleType(gen, (pytd.NamedType("str"), pytd.NamedType("float")))
self.assertRaises(visitors.ContainerError,
lambda: t2.Visit(visitors.VerifyContainers()))
# Okay
param = pytd.TypeParameter("T")
parent = pytd.GenericType(gen, (param,))
base.cls = pytd.Class(
"tuple", None, (parent,), (), (), (), None, (pytd.TemplateItem(param),))
t3 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float")))
t3.Visit(visitors.VerifyContainers())
def generate_ast(self):
"""Generate this class's AST, including updated members."""
return pytd.Class(name=self.name,
metaclass=self.pytd_cls.metaclass,
bases=self.pytd_cls.bases,
methods=tuple(self._member_map[m.name]
for m in self.pytd_cls.methods),
constants=self.pytd_cls.constants,
classes=self.pytd_cls.classes,
decorators=self.pytd_cls.decorators,
slots=self.pytd_cls.slots,
template=self.pytd_cls.template)
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
# class-level attributes
for name, member in v.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(v.vm.exitpoint):
if isinstance(value, abstract.Function):
val = self.value_to_pytd_def(node, value, name)
if isinstance(val, pytd.Function):
methods[name] = val
elif isinstance(v, pytd.TYPE):
constants[name].add_type(val)
else:
raise AssertionError(str(type(val)))
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in v.instances:
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(v.vm.exitpoint):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
bases = [pytd_utils.JoinTypes(b.get_instance_type(node)
for b in basevar.data)
for basevar in v.bases()
if basevar is not v.vm.convert.oldstyleclass_type]
constants = [pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
return pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
template=())
def _typed_dict_to_def(self, node, v, name):
constants = []
for k, var in v.props.fields.items():
typ = pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, set(), {})
for p in var.data)
constants.append(pytd.Constant(k, typ))
bases = (pytd.NamedType("typing.TypedDict"), )
return pytd.Class(name=name,
metaclass=None,
bases=bases,
methods=(),
constants=tuple(constants),
classes=(),
decorators=(),
slots=None,
template=())
def VisitClass(self, node):
# If we have a dataclass-like decorator we need to preserve the order of the
# class attributes, otherwise inheritance will not work correctly.
if any(x.name in ("attr.s", "dataclasses.dataclass")
for x in node.decorators):
constants = node.constants
else:
constants = sorted(node.constants)
return pytd.Class(name=node.name,
metaclass=node.metaclass,
bases=node.bases,
methods=tuple(sorted(node.methods)),
constants=tuple(constants),
decorators=tuple(sorted(node.decorators)),
classes=tuple(sorted(node.classes)),
slots=tuple(sorted(node.slots))
if node.slots is not None else None,
template=node.template)
def new_new_type(self, name, typ):
"""Returns a type for a NewType."""
args = [("self", pytd.AnythingType()), ("val", typ)]
ret = pytd.NamedType("NoneType")
methods = function.merge_method_signatures(
[function.NameAndSig.make("__init__", args, ret)])
cls_name = escape.pack_newtype_base_class(
name, len(self.generated_classes[name]))
cls = pytd.Class(name=cls_name,
metaclass=None,
parents=(typ, ),
methods=tuple(methods),
constants=(),
decorators=(),
classes=(),
slots=None,
template=())
self.generated_classes[name].append(cls)
return pytd.NamedType(cls_name)
def pytd_classes_for_call_traces(self):
class_to_records = collections.defaultdict(list)
for call_record in self._method_calls:
args = call_record.positional_arguments
if not any(isinstance(a.data, abstract.Unknown) for a in args):
# We don't need to record call signatures that don't involve
# unknowns - there's nothing to solve for.
continue
clsvar = args[0].data.get_class()
for cls in clsvar.data:
if isinstance(cls, abstract.PyTDClass):
class_to_records[cls].append(call_record)
classes = []
for cls, call_records in class_to_records.items():
classes.append(pytd.Class(
name="~" + cls.name,
parents=(), # not used in solver
methods=self._call_traces_to_function(call_records),
constants=(),
template=(),
))
return classes
def p_classdef(self, p):
"""classdef : CLASS NAME class_parents COLON maybe_class_funcs end_class"""
_, _, name, (template, parents), _, class_funcs, _ = p
methoddefs = [x for x in class_funcs if isinstance(x, NameAndSig)]
constants = [x for x in class_funcs if isinstance(x, pytd.Constant)]
if (set(f.name for f in methoddefs) | set(c.name for c in constants) !=
set(d.name for d in class_funcs)):
# TODO(kramm): raise a syntax error right when the identifier is defined.
raise make_syntax_error(self, 'Duplicate identifier(s)', p)
template_names = {t.name for t in template}
for _, sig, _ in methoddefs:
for t in sig.template:
if t.name in template_names:
raise make_syntax_error(
self, 'Duplicate template parameter %s' % t.name, p)
cls = pytd.Class(name=name,
parents=parents,
methods=tuple(self.MergeSignatures(methoddefs)),
constants=tuple(constants),
template=template)
p[0] = cls.Visit(visitors.AdjustSelf())
def VisitClass(self, node):
return pytd.Class(name=node.name,
parents=node.parents,
methods=tuple(sorted(node.methods)),
constants=tuple(sorted(node.constants)),
template=node.template)
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = self.get_annotations_dict(v.members)
for name, t in self.uninitialized_annotations_to_instance_types(
node, annots, v.members):
constants[name].add_type(t)
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE:
continue
for value, is_annotation in self.get_annotated_values(
node, name, member, annots):
if is_annotation:
constants[name].add_type(value)
continue
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(node, value.fget):
constants[name].add_type(typ)
else:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.StaticMethodInstance):
try:
methods[name] = self._static_method_to_def(
node, value, name, pytd.STATICMETHOD)
except abstract_utils.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.ClassMethodInstance):
try:
methods[name] = self._class_method_to_def(
node, value, name, pytd.CLASSMETHOD)
except abstract_utils.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, abstract.Function):
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = self.value_to_pytd_def(node, value, name).Visit(
visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes(node, [value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in set(v.instances):
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(self.vm.exitpoint, strict=False):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(pytd_utils.JoinTypes(b.get_instance_type(node)
for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [self._class_to_def(node, x, x.name)
for x in v.get_inner_classes()]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
return cls
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
self._scopes.append(class_name)
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members)
annotated_names = set()
def add_constants(iterator):
for name, t in iterator:
if t is None:
# Remove the entry from constants
annotated_names.add(name)
elif name not in annotated_names:
constants[name].add_type(t)
annotated_names.add(name)
add_constants(
self._ordered_attrs_to_instance_types(node, v.metadata, annots))
add_constants(self.annotations_to_instance_types(node, annots))
def get_decorated_method(name, value, func_slot):
fvar = getattr(value, func_slot)
func = abstract_utils.get_atomic_value(fvar, abstract.Function)
defn = self.value_to_pytd_def(node, func, name)
defn = defn.Visit(visitors.DropMutableParameters())
return defn
def add_decorated_method(name, value, kind):
try:
defn = get_decorated_method(name, value, "func")
except (AttributeError, abstract_utils.ConversionError):
constants[name].add_type(pytd.AnythingType())
return
defn = defn.Replace(kind=kind)
methods[name] = defn
# If decorators are output as aliases to NamedTypes, they will be converted
# to Functions and fail a verification step if those functions have type
# parameters. Since we just want the function name, and since we have a
# fully resolved name at this stage, we just output a minimal pytd.Function
sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ())
decorators = [
pytd.Alias(x, pytd.Function(x, (sig, ), pytd.MethodTypes.METHOD,
0)) for x in v.decorators
]
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE or name in annotated_names:
continue
for value in member.FilteredData(self.vm.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(
pytd.Annotated(typ, ("'property'", )))
else:
constants[name].add_type(
pytd.Annotated(pytd.AnythingType(),
("'property'", )))
elif isinstance(value, special_builtins.StaticMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.STATICMETHOD)
elif isinstance(value, special_builtins.ClassMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.CLASSMETHOD)
elif isinstance(value, abstract.Function):
# value_to_pytd_def returns different pytd node types depending on the
# input type, which pytype struggles to reason about.
method = cast(pytd.Function,
self.value_to_pytd_def(node, value, name))
keep = lambda name: not name or name.startswith(v.name)
signatures = tuple(
s for s in method.signatures
if not s.params or keep(s.params[0].type.name))
if signatures and signatures != method.signatures:
# Filter out calls made from subclasses unless they are the only
# ones recorded; when inferring types for ParentClass.__init__, we
# do not want `self: Union[ParentClass, Subclass]`.
method = method.Replace(signatures=signatures)
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = method.Visit(
visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes([value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# Instance-level attributes: all attributes from 'canonical' instances (that
# is, ones created by analyze.py:analyze_class()) are added. Attributes from
# non-canonical instances are added if their canonical values do not contain
# type parameters.
ignore = set(annotated_names)
canonical_attributes = set()
def add_attributes_from(instance):
for name, member in instance.members.items():
if name in CLASS_LEVEL_IGNORE or name in ignore:
continue
for value in member.FilteredData(self.vm.exitpoint,
strict=False):
typ = value.to_type(node)
if pytd_utils.GetTypeParameters(typ):
# This attribute's type comes from an annotation that contains a
# type parameter; we do not want to merge in substituted values of
# the type parameter.
canonical_attributes.add(name)
constants[name].add_type(typ)
for instance in v.canonical_instances:
add_attributes_from(instance)
ignore |= canonical_attributes
for instance in v.instances - v.canonical_instances:
add_attributes_from(instance)
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
decorators=tuple(decorators),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
self._scopes.pop()
return cls
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE:
continue
for value in member.FilteredData(v.vm.exitpoint):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
for typ in self._property_to_types(node, value):
constants[name].add_type(typ)
elif isinstance(value, special_builtins.StaticMethodInstance):
try:
methods[name] = self._static_method_to_def(
node, value, name, pytd.STATICMETHOD)
except abstract.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, special_builtins.ClassMethodInstance):
try:
methods[name] = self._class_method_to_def(
node, value, name, pytd.CLASSMETHOD)
except abstract.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, abstract.Function):
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = self.value_to_pytd_def(
node, value,
name).Visit(visitors.DropMutableParameters())
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in v.instances:
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(v.vm.exitpoint):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
bases = [
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data)
for basevar in v.bases()
if basevar.data != [v.vm.convert.oldstyleclass_type]
]
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
return pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
slots=v.slots,
template=())
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE:
continue
for value in member.FilteredData(v.vm.exitpoint):
if self._is_instance(value, "__builtin__.property"):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
for typ in self._property_to_types(node, value):
constants[name].add_type(typ)
elif self._is_instance(value, "__builtin__.staticmethod"):
try:
methods[name] = self._special_method_to_def(
node, value, name, pytd.STATICMETHOD)
except abstract.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif self._is_instance(value, "__builtin__.classmethod"):
try:
methods[name] = self._special_method_to_def(
node, value, name, pytd.CLASSMETHOD)
except abstract.ConversionError:
constants[name].add_type(pytd.AnythingType())
elif isinstance(value, abstract.Function):
methods[name] = self.value_to_pytd_def(node, value, name)
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in v.instances:
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(v.vm.exitpoint):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
bases = [
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data)
for basevar in v.bases()
if basevar.data != [v.vm.convert.oldstyleclass_type]
]
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
return pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
template=())
def add_class(self, class_name, parent_args, defs):
"""Add a class to the module.
Args:
class_name: The name of the class (a string).
parent_args: A list of parent types and (keyword, value) tuples.
Parent types must be instances of pytd.Type. Keyword tuples must
appear at the end of the list. Currently the only supported keyword
is 'metaclass'.
defs: A list of constant (pytd.Constant) and function (_NameAndSig)
definitions.
Raises:
ParseError: if defs contains duplicate names (excluding multiple
definitions of a function, which is allowed).
"""
# Process parent_args, extracting parents and possibly a metaclass.
parents = []
metaclass = None
for i, p in enumerate(parent_args):
if isinstance(p, pytd.Type):
parents.append(p)
else:
keyword, value = p
if i != len(parent_args) - 1:
raise ParseError("metaclass must be last argument")
if keyword != "metaclass":
raise ParseError(
"Only 'metaclass' allowed as classdef kwarg")
metaclass = value
constants, methods = _split_definitions(defs)
all_names = (list(set(f.name
for f in methods)) + [c.name for c in constants])
duplicates = [
name for name, count in collections.Counter(all_names).items()
if count >= 2
]
if duplicates:
# TODO(kramm): raise a syntax error right when the identifier is defined.
raise ParseError("Duplicate identifier(s): " +
", ".join(duplicates))
# This check is performed after the above error checking so that errors
# will be spotted even in non-active conditional code.
if not self._current_condition.active:
return
# TODO(dbaum): Is NothingType even legal here? The grammar accepts it but
# perhaps it should be a ParseError.
parents = [p for p in parents if not isinstance(p, pytd.NothingType)]
methods, properties = _merge_signatures(methods)
# Ensure that old style classes inherit from classobj.
if not parents and class_name not in ["classobj", "object"]:
parents = (pytd.NamedType("classobj"), )
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(parents),
methods=tuple(methods),
constants=tuple(constants + properties),
template=())
self._classes.append(cls)
def build_class(self, class_name, bases, keywords, decorators,
defs) -> pytd.Class:
"""Build a pytd.Class from definitions collected from an ast node."""
parents, namedtuple_index = classdef.get_parents(bases)
metaclass = classdef.get_metaclass(keywords, parents)
constants, methods, aliases, slots, classes = _split_definitions(defs)
# Make sure we don't have duplicate definitions.
classdef.check_for_duplicate_defs(methods, constants, aliases)
# Generate a NamedTuple proxy base class if needed
if namedtuple_index is not None:
namedtuple_parent = self.new_named_tuple(class_name,
[(c.name, c.type)
for c in constants])
parents[namedtuple_index] = namedtuple_parent
constants = []
if aliases:
vals_dict = {
val.name: val
for val in constants + aliases + methods + classes
}
for val in aliases:
name = val.name
seen_names = set()
while isinstance(val, pytd.Alias):
if isinstance(val.type, pytd.NamedType):
_, _, base_name = val.type.name.rpartition(".")
if base_name in seen_names:
# This happens in cases like:
# class X:
# Y = something.Y
# Since we try to resolve aliases immediately, we don't know what
# type to fill in when the alias value comes from outside the
# class. The best we can do is Any.
val = pytd.Constant(name, pytd.AnythingType())
continue
seen_names.add(base_name)
if base_name in vals_dict:
val = vals_dict[base_name]
continue
# The alias value comes from outside the class. The best we can do is
# to fill in Any.
val = pytd.Constant(name, pytd.AnythingType())
if isinstance(val, function.NameAndSig):
val = dataclasses.replace(val, name=name)
methods.append(val)
else:
if isinstance(val, pytd.Class):
t = pytdgen.pytd_type(
pytd.NamedType(class_name + "." + val.name))
else:
t = val.type
constants.append(pytd.Constant(name, t))
parents = [p for p in parents if not isinstance(p, pytd.NothingType)]
methods = function.merge_method_signatures(methods)
if not parents and class_name not in ["classobj", "object"]:
# A parent-less class inherits from classobj in Python 2 and from object
# in Python 3. typeshed assumes the Python 3 behavior for all stubs, so we
# do the same here.
parents = (pytd.NamedType("object"), )
return pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(parents),
methods=tuple(methods),
constants=tuple(constants),
classes=tuple(classes),
decorators=tuple(decorators),
slots=slots,
template=())
def testClearClassPointers(self):
cls = pytd.Class("foo", None, (), (), (), (), None, ())
t = pytd.ClassType("foo", cls)
t = t.Visit(visitors.ClearClassPointers())
self.assertIsNone(t.cls)
