def testAdjustTypeParametersWithBuiltins(self):
ast = self.ParseWithBuiltins("""
T = TypeVar("T")
K = TypeVar("K")
V = TypeVar("V")
class Foo(List[int]): pass
class Bar(Dict[T, int]): pass
class Baz(Generic[K, V]): pass
class Qux(Baz[str, int]): pass
""")
foo = ast.Lookup("Foo")
bar = ast.Lookup("Bar")
qux = ast.Lookup("Qux")
foo_parent, = foo.parents
bar_parent, = bar.parents
qux_parent, = qux.parents
# Expected:
# Class(Foo, parent=GenericType(List, parameters=(int,)), template=())
# Class(Bar, parent=GenericType(Dict, parameters=(T, int)), template=(T))
# Class(Qux, parent=GenericType(Baz, parameters=(str, int)), template=())
self.assertEqual((pytd.ClassType("int"),), foo_parent.parameters)
self.assertEqual((), foo.template)
self.assertEqual(
(pytd.TypeParameter("T", scope="Bar"), pytd.ClassType("int")),
bar_parent.parameters)
self.assertEqual(
(pytd.TemplateItem(pytd.TypeParameter("T", scope="Bar")),),
bar.template)
self.assertEqual((pytd.ClassType("str"), pytd.ClassType("int")),
qux_parent.parameters)
self.assertEqual((), qux.template)
def testConcatTypeParameters(self):
"""Test for concatenating ASTs with type parameters."""
ast1 = self.Parse("""T = TypeVar("T")""", name="__builtin__")
ast2 = self.Parse("""T = TypeVar("T")""")
combined = pytd_utils.Concat(ast1, ast2)
self.assertEqual(combined.Lookup("__builtin__.T"),
pytd.TypeParameter("T", scope="__builtin__"))
self.assertEqual(combined.Lookup("T"), pytd.TypeParameter("T", scope=None))
def testAdjustTypeParametersWithDuplicates(self):
ast = self.ParseWithBuiltins("""
T = TypeVar("T")
class A(Dict[T, T], Generic[T]): pass
""")
a = ast.Lookup("A")
self.assertEqual(
(pytd.TemplateItem(pytd.TypeParameter("T", (), None, "A")),
pytd.TemplateItem(pytd.TypeParameter("T",
(), None, "A"))), a.template)
def SplitParents(parser, p, parents):
"""Strip the special Generic[...] class out of the base classes."""
template = ()
other_parents = []
for parent in parents:
if (isinstance(parent, pytd.GenericType) and parent.base_type
== pytd.ExternalType('Generic', 'typing')):
if not all(
isinstance(param, pytd.NamedType)
for param in parent.parameters):
make_syntax_error(
parser,
'Illegal template parameter %s' % pytd.Print(parent), p)
if template:
make_syntax_error(parser, 'Duplicate Template base class', p)
template = tuple(
pytd.TemplateItem(pytd.TypeParameter(param.name))
for param in parent.parameters)
all_names = [t.name for t in template]
duplicates = [
name for name, count in collections.Counter(all_names).items()
if count >= 2
]
if duplicates:
make_syntax_error(
parser,
'Duplicate template parameters' + ', '.join(duplicates), p)
else:
if parent != pytd.NothingType():
other_parents.append(parent)
return template, tuple(other_parents)
def value_to_pytd_def(self, node, v, name):
"""Get a PyTD definition for this object.
Args:
node: The node.
v: The object.
name: The object name.
Returns:
A PyTD definition.
"""
if isinstance(v, abstract.PyTDFunction):
return pytd.Function(
name, tuple(sig.pytd_sig for sig in v.signatures), pytd.METHOD)
elif isinstance(v, abstract.InterpreterFunction):
return self._function_to_def(node, v, name)
elif isinstance(v, abstract.ParameterizedClass):
return pytd.Alias(name, v.get_instance_type(node))
elif isinstance(v, abstract.PyTDClass):
# This happens if a module does e.g. "from x import y as z", i.e., copies
# something from another module to the local namespace. We *could*
# reproduce the entire class, but we choose a more dense representation.
return v.to_type(node)
elif isinstance(v, abstract.InterpreterClass):
return self._class_to_def(node, v, name)
elif isinstance(v, abstract.TypeVariable):
return pytd.TypeParameter(name, None)
elif isinstance(v, abstract.Unsolvable):
return pytd.Constant(name, v.to_type(node))
else:
raise NotImplementedError(v.__class__.__name__)
def _parameterized_type(self, base_type: Any, parameters):
"""Return a parameterized type."""
if self._matches_named_type(base_type, _LITERAL_TYPES):
return pytd_literal(parameters)
elif self._matches_named_type(base_type, _ANNOTATED_TYPES):
return pytd_annotated(parameters)
elif self._matches_named_type(base_type, _FINAL_TYPES):
typ, = parameters
return pytd.GenericType(pytd.NamedType("typing.Final"), (typ, ))
elif self._matches_named_type(base_type, _TYPEGUARD_TYPES):
# We do not yet support PEP 647, User-Defined Type Guards. To avoid
# blocking typeshed, convert type guards to plain bools.
return pytd.NamedType("bool")
elif any(isinstance(p, types.Pyval) for p in parameters):
parameters = ", ".join(
p.repr_str() if isinstance(p, types.Pyval) else "_"
for p in parameters)
raise ParseError("%s[%s] not supported" %
(pytd_utils.Print(base_type), parameters))
elif pytdgen.is_any(base_type):
return pytd.AnythingType()
elif len(parameters) == 2 and parameters[-1] is self.ELLIPSIS and (
not self._matches_named_type(base_type, _CALLABLE_TYPES)):
element_type = parameters[0]
if element_type is self.ELLIPSIS:
raise ParseError("[..., ...] not supported")
return pytd.GenericType(base_type=base_type,
parameters=(element_type, ))
else:
processed_parameters = []
# We do not yet support PEP 612, Parameter Specification Variables.
# To avoid blocking typeshed from adopting this PEP, we convert new
# features to approximations that only use supported features.
for p in parameters:
if p is self.ELLIPSIS:
processed = pytd.AnythingType()
elif (p in self.param_specs and self._matches_full_name(
base_type, "typing.Generic")):
# Replacing a ParamSpec with a TypeVar isn't correct, but it'll work
# for simple cases in which the filled value is also a ParamSpec.
if not any(t.name == p.name for t in self.type_params):
self.type_params.append(pytd.TypeParameter(p.name))
processed = p
elif (p in self.param_specs
or (isinstance(p, pytd.GenericType)
and self._matches_full_name(p, _CONCATENATE_TYPES))):
processed = pytd.AnythingType()
else:
processed = p
processed_parameters.append(processed)
parameters = tuple(processed_parameters)
if self._matches_named_type(base_type, _TUPLE_TYPES):
return pytdgen.heterogeneous_tuple(base_type, parameters)
elif self._matches_named_type(base_type, _CALLABLE_TYPES):
return pytdgen.pytd_callable(base_type, parameters)
else:
assert parameters
return pytd.GenericType(base_type=base_type,
parameters=parameters)
def add_type_var(self, name, typevar):
"""Add a type variable, <name> = TypeVar(<name_arg>, <args>)."""
if name != typevar.name:
raise ParseError("TypeVar name needs to be %r (not %r)" % (
typevar.name, name))
bound = typevar.bound
if isinstance(bound, str):
bound = pytd.NamedType(bound)
constraints = tuple(typevar.constraints) if typevar.constraints else ()
self.type_params.append(pytd.TypeParameter(
name=name, constraints=constraints, bound=bound))
def testAdjustTypeParameters(self):
ast = self.Parse("""
T = TypeVar("T")
T2 = TypeVar("T2")
def f(x: T) -> T
class A(Generic[T]):
def a(self, x: T2) -> None:
self = A[T or T2]
""")
f = ast.Lookup("f")
sig, = f.signatures
p_x, = sig.params
self.assertEqual(sig.template,
(pytd.TemplateItem(pytd.TypeParameter("T", scope="f")),))
self.assertEqual(p_x.type, pytd.TypeParameter("T", scope="f"))
cls = ast.Lookup("A")
f_cls, = cls.methods
sig_cls, = f_cls.signatures
p_self, p_x_cls = sig_cls.params
self.assertEqual(cls.template,
(pytd.TemplateItem(pytd.TypeParameter("T", scope="A")),))
self.assertEqual(sig_cls.template, (pytd.TemplateItem(
pytd.TypeParameter("T2", scope="A.a")),))
self.assertEqual(p_self.type.parameters,
(pytd.TypeParameter("T", scope="A"),))
self.assertEqual(p_x_cls.type, pytd.TypeParameter("T2", scope="A.a"))
def new_type(self,
name: Union[str, pytd_node.Node],
parameters: Optional[List[pytd.Type]] = None) -> pytd.Type:
"""Return the AST for a type.
Args:
name: The name of the type.
parameters: List of type parameters.
Returns:
A pytd type node.
Raises:
ParseError: if the wrong number of parameters is supplied for the
base_type - e.g., 2 parameters to Optional or no parameters to Union.
"""
base_type = self.resolve_type(name)
for p in self.param_specs:
if base_type.name.startswith(f"{p.name}."):
_, attr = base_type.name.split(".", 1)
if attr not in ("args", "kwargs"):
raise ParseError(
f"Unrecognized ParamSpec attribute: {attr}")
# We do not yet support typing.ParamSpec, so replace references to its
# args and kwargs attributes with Any.
return pytd.AnythingType()
if not isinstance(base_type, pytd.NamedType):
# We assume that all type parameters have been defined. Since pytype
# orders type parameters to appear before classes and functions, this
# assumption is generally safe. AnyStr is special-cased because imported
# type parameters aren't recognized.
type_params = self.type_params + [
pytd.TypeParameter("typing.AnyStr")
]
base_type = base_type.Visit(_InsertTypeParameters(type_params))
try:
resolved_type = visitors.MaybeSubstituteParameters(
base_type, parameters)
except ValueError as e:
raise ParseError(str(e)) from e
if resolved_type:
return resolved_type
if parameters is not None:
if (len(parameters) > 1 and isinstance(base_type, pytd.NamedType)
and base_type.name == "typing.Optional"):
raise ParseError("Too many options to %s" % base_type.name)
return self._parameterized_type(base_type, parameters)
else:
if (isinstance(base_type, pytd.NamedType)
and base_type.name in _TYPING_SETS):
raise ParseError("Missing options to %s" % base_type.name)
return base_type
def testDiffPickleAst(self):
ast1 = pytd.TypeDeclUnit("foo", (), (), (), (), ())
ast2 = ast1.Replace(type_params=(pytd.TypeParameter("T", (), None, None),))
with file_utils.Tempdir() as d:
data = []
for ast in (ast1, ast2):
filename = os.path.join(d.path, "foo.pickled")
serialize_ast.StoreAst(ast, filename)
with open(filename, "rb") as fi:
data.append(fi.read())
named_pickles1 = [("foo", data[0])]
named_pickles2 = [("foo", data[1])]
self.assertTrue(pytd_utils.DiffNamedPickles(named_pickles1, named_pickles2))
def value_instance_to_pytd_type(self, node, v, instance, seen, view):
"""Get the PyTD type an instance of this object would have.
Args:
node: The node.
v: The object.
instance: The instance.
seen: Already seen instances.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, abstract.Union):
return pytd.UnionType(tuple(
self.value_instance_to_pytd_type(node, t, instance, seen, view)
for t in v.options))
elif isinstance(v, abstract.AnnotationContainer):
return self.value_instance_to_pytd_type(
node, v.base_cls, instance, seen, view)
elif isinstance(v, abstract.Class):
if v.official_name is None:
return pytd.AnythingType()
if seen is None:
seen = set()
if instance in seen:
# We have a circular dependency in our types (e.g., lst[0] == lst). Stop
# descending into the type parameters.
type_params = ()
else:
type_params = tuple(t.name for t in v.template)
if instance is not None:
seen.add(instance)
type_arguments = self._value_to_parameter_types(
node, v, instance, type_params, seen, view)
base = pytd_utils.NamedTypeWithModule(v.official_name, v.module)
if self._is_tuple(v, instance):
if type_arguments:
homogeneous = False
else:
homogeneous = True
type_arguments = [pytd.NothingType()]
else:
homogeneous = len(type_arguments) == 1
return pytd_utils.MakeClassOrContainerType(
base, type_arguments, homogeneous)
elif isinstance(v, abstract.TypeVariable):
return pytd.TypeParameter(v.name, None)
else:
log.info("Using ? for instance of %s", v.name)
return pytd.AnythingType()
def testTypeParameters(self):
"""Test parsing of type parameters."""
src = textwrap.dedent("""
T = TypeVar("T")
T2 = TypeVar("T2")
def f(x: T) -> T
class A(Generic[T]):
def a(self, x: T2) -> None:
self := A[T or T2]
""")
tree = parser.TypeDeclParser().Parse(src)
param1 = tree.Lookup("T")
param2 = tree.Lookup("T2")
self.assertEquals(param1, pytd.TypeParameter("T", None))
self.assertEquals(param2, pytd.TypeParameter("T2", None))
self.assertEquals(tree.type_params, (param1, param2))
f = tree.Lookup("f")
sig, = f.signatures
p_x, = sig.params
self.assertEquals(p_x.type, pytd.TypeParameter("T", None))
cls = tree.Lookup("A")
cls_parent, = cls.parents
f_cls, = cls.methods
sig_cls, = f_cls.signatures
# AdjustSelf has not been called yet, so self may not have the right type
_, p_x_cls = sig_cls.params
self.assertEquals(cls_parent.parameters,
(pytd.TypeParameter("T", None), ))
self.assertEquals(p_x_cls.type, pytd.TypeParameter("T2", None))
# The parser should not have attempted to insert templates! It does
# not know about imported type parameters.
self.assertEquals(sig.template, ())
self.assertEquals(cls.template, ())
self.assertEquals(sig_cls.template, ())
def add_type_var(self, name, param_list):
"""Add a type variable with the given name and parameter list."""
params = _validate_params(param_list)
if (not params.required
or not isinstance(params.required[0], pytd.Parameter)):
raise ParseError("TypeVar's first arg should be a string")
# Allow and ignore any other arguments (types, covariant=..., etc)
name_param = params.required[0].name
if name != name_param:
raise ParseError("TypeVar name needs to be %r (not %r)" %
(name_param, name))
if not self._current_condition.active:
return
self._type_params.append(pytd.TypeParameter(name, scope=None))
def testVerifyHeterogeneousTuple(self):
# Error: does not inherit from Generic
base = pytd.ClassType("tuple")
base.cls = pytd.Class("tuple", None, (), (), (), (), None, ())
t1 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float")))
self.assertRaises(visitors.ContainerError,
lambda: t1.Visit(visitors.VerifyContainers()))
# Error: Generic[str, float]
gen = pytd.ClassType("typing.Generic")
gen.cls = pytd.Class("typing.Generic", None, (), (), (), (), None, ())
t2 = pytd.TupleType(gen, (pytd.NamedType("str"), pytd.NamedType("float")))
self.assertRaises(visitors.ContainerError,
lambda: t2.Visit(visitors.VerifyContainers()))
# Okay
param = pytd.TypeParameter("T")
parent = pytd.GenericType(gen, (param,))
base.cls = pytd.Class(
"tuple", None, (parent,), (), (), (), None, (pytd.TemplateItem(param),))
t3 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float")))
t3.Visit(visitors.VerifyContainers())
def add_type_var(self, name, name_arg, args):
"""Add a type variable, <name> = TypeVar(<name_arg>, <args>)."""
if name != name_arg:
raise ParseError("TypeVar name needs to be %r (not %r)" % (
name_arg, name))
# 'bound' is the only keyword argument we currently use.
# TODO(rechen): We should enforce the PEP 484 guideline that
# len(constraints) != 1. However, this guideline is currently violated
# in typeshed (see https://github.com/python/typeshed/pull/806).
constraints, named_args = args
named_args = dict(named_args) if named_args else {}
extra = set(named_args) - {"bound", "covariant", "contravariant"}
if extra:
raise ParseError("Unrecognized keyword(s): %s" % ", ".join(extra))
if not self._current_condition.active:
return
self._type_params.append(pytd.TypeParameter(
name=name,
constraints=() if constraints is None else tuple(constraints),
bound=named_args.get("bound")))
def _namedtuple_new(self, name, fields):
"""Build a __new__ method for a namedtuple with the given fields.
For a namedtuple defined as NamedTuple("_", [("foo", int), ("bar", str)]),
generates the method
def __new__(cls: Type[_T], foo: int, bar: str) -> _T: ...
where _T is a TypeVar bounded by the class type.
Args:
name: The class name.
fields: A list of (name, type) pairs representing the namedtuple fields.
Returns:
A _NameAndSig object for a __new__ method.
"""
type_param = pytd.TypeParameter("_T" + name, bound=pytd.NamedType(name))
self._type_params.append(type_param)
cls_arg = (
"cls", pytd.GenericType(pytd.NamedType("type"), (type_param,)), None)
args = [cls_arg] + [(n, t, None) for n, t in fields]
return self.new_function((), "__new__", args, type_param, ())
def _make_new(self, name: str,
fields: List[Tuple[str, Any]]) -> function.NameAndSig:
"""Build a __new__ method for a namedtuple with the given fields.
For a namedtuple defined as NamedTuple("_", [("foo", int), ("bar", str)]),
generates the method
def __new__(cls: Type[_T], foo: int, bar: str) -> _T: ...
where _T is a TypeVar bounded by the class type.
Args:
name: The class name.
fields: A list of (name, type) pairs representing the namedtuple fields.
Returns:
A function.NameAndSig object for a __new__ method.
"""
type_param = pytd.TypeParameter("_T" + name,
bound=pytd.NamedType(name))
self.type_param = type_param
cls_arg = ("cls", pytdgen.pytd_type(type_param))
args = [cls_arg] + fields
return function.NameAndSig.make("__new__", args, type_param)
def p_funcdef(self, p):
"""funcdef : DEF NAME LPAREN params RPAREN return raises signature maybe_body"""
_, _, name, _, params, _, return_type, raises, _, body = p
# TODO(kramm): Output a warning if we already encountered a signature
# with these types (but potentially different argument names)
if name == '__init__' and isinstance(return_type, pytd.AnythingType):
ret = pytd.NamedType('NoneType')
else:
ret = return_type
signature = pytd.Signature(params=tuple(params.required),
return_type=ret,
exceptions=tuple(raises),
template=(),
has_optional=params.has_optional)
typeparams = {
name: pytd.TypeParameter(name)
for name in self.context.typevars
}
used_typeparams = set()
signature = signature.Visit(
visitors.ReplaceTypes(typeparams, used_typeparams))
if used_typeparams:
signature = signature.Replace(template=tuple(
pytd.TemplateItem(typeparams[name])
for name in used_typeparams))
for mutator in body:
try:
signature = signature.Visit(mutator)
except NotImplementedError as e:
make_syntax_error(self, e.message, p)
if not mutator.successful:
make_syntax_error(self, 'No parameter named %s' % mutator.name,
p)
p[0] = NameAndSig(name=name, signature=signature, external_code=False)
def _typeparam_to_def(self, node, v, name): constraints = tuple(c.get_instance_type(node) for c in v.constraints) bound = v.bound and v.bound.get_instance_type(node) return pytd.TypeParameter(name, constraints=constraints, bound=bound)
