def convert_function_annotations(self, node, raw_annotations):
if raw_annotations:
# {"i": int, "return": str} is stored as (int, str, ("i", "return"))
names = abstract.get_atomic_python_constant(raw_annotations[-1])
type_list = raw_annotations[:-1]
annotations = {}
late_annotations = {}
for name, t in zip(names, type_list):
name = abstract.get_atomic_python_constant(name)
visible = t.Data(node)
if len(visible) > 1:
self.vm.errorlog.ambiguous_annotation(
self.vm.frames, visible, name)
else:
try:
annot = self._process_one_annotation(
visible[0], name, self.vm.frames)
except self.LateAnnotationError:
late_annotations[name] = LateAnnotation(
visible[0], name, self.vm.simple_stack())
else:
if annot is not None:
annotations[name] = annot
return annotations, late_annotations
else:
return {}, {}
def _getargs(self, node, args):
self._match_args(node, args)
# Normally we would use typing.NamedTuple.__new__ to match args to
# parameters, but we can't import typing.
# TODO(tsudol): Replace with typing.NamedTuple.__new__.
f = function.Signature.from_param_names("typing.NamedTuple",
["typename", "fields"])
callargs = {
arg_name: arg_var
for arg_name, arg_var, _ in f.iter_args(args)
}
# typing.NamedTuple doesn't support rename or verbose
name_var = callargs["typename"]
fields_var = callargs["fields"]
fields = abstract.get_atomic_python_constant(fields_var)
# The fields is a list of tuples, so we need to deeply unwrap them.
fields = [abstract.get_atomic_python_constant(t) for t in fields]
# We need the actual string for the field names and the AtomicAbstractValue
# for the field types.
names = []
types = []
for field in fields:
if (len(field) != 2 or any(not self._is_str_instance(v)
for v in field[0].data)):
# Note that we don't need to check field[1] because both 'str'
# (forward reference) and 'type' are valid for it.
sig, = self.signatures
bad_param = abstract.BadParam(name="fields",
expected=self._fields_type)
raise abstract.WrongArgTypes(sig.signature, args, self.vm,
bad_param)
name, typ = field
names.append(abstract.get_atomic_python_constant(name))
types.append(abstract.get_atomic_value(typ))
return name_var, names, types
def _getargs(self, node, args):
self.match_args(node, args)
sig, = self.signatures
callargs = {
name: var
for name, var, _ in sig.signature.iter_args(args)
}
# typing.NamedTuple doesn't support rename or verbose
name_var = callargs["typename"]
fields_var = callargs["fields"]
fields = abstract.get_atomic_python_constant(fields_var)
# The fields is a list of tuples, so we need to deeply unwrap them.
fields = [abstract.get_atomic_python_constant(t) for t in fields]
# We need the actual string for the field names and the AtomicAbstractValue
# for the field types.
names = []
types = []
for field in fields:
if (len(field) != 2 or any(not self._is_str_instance(v)
for v in field[0].data)):
# Note that we don't need to check field[1] because both 'str'
# (forward reference) and 'type' are valid for it.
sig, = self.signatures
bad_param = abstract.BadParam(name="fields",
expected=self._fields_type)
raise abstract.WrongArgTypes(sig.signature, args, self.vm,
bad_param)
name, typ = field
names.append(abstract.get_atomic_python_constant(name))
types.append(abstract.get_atomic_value(typ))
return name_var, names, types
def _getargs(self, node, args):
"""Extracts the typename, field_names and rename arguments.
collections.namedtuple takes potentially 4 arguments, but we only care about
three of them. This function checks the argument count and ensures multiple
values aren't passed for 'verbose' and 'rename'.
Args:
node: The current CFG node. Used by _match_args.
args: An abstract.FunctionArgs object
Returns:
A tuple containing the typename, field_names and rename arguments passed
to this call to collections.namedtuple.
Raises:
abstract.FailedFunctionCall: The arguments do not match those needed by
the function call. See also: abstract.PyTDFunction._match_args().
abstract.ConversionError: One of the arguments could not be extracted.
Typically occurs if typename or one of the field names is in unicode.
"""
# abstract.PyTDFunction._match_args checks the args for this call.
self._match_args(node, args)
# inspect.callargs returns a dictionary mapping the argument names to
# the values in args.posargs and args.namedargs (or False if there is no
# value given).
callargs = inspect.getcallargs(collections.namedtuple, *args.posargs,
**args.namedargs)
# The name of the namedtuple class is the first arg (a Variable)
# We need the actual Variable later, so we'll just return name_var and
# extract the name itself later.
name_var = callargs["typename"]
# The fields are also a Variable, which stores the field names as Variables.
# Extract the list itself, we don't need the wrapper.
fields_var = callargs["field_names"]
fields = abstract.get_atomic_python_constant(fields_var)
# namedtuple fields can be given as a single string, e.g. "a, b, c" or as a
# list [Variable('a'), Variable('b'), Variable('c')].
# We just want a list of strings.
if isinstance(fields, (str, unicode)):
field_names = fields.replace(",", " ").split()
else:
field_names = [
abstract.get_atomic_python_constant(f) for f in fields
]
# namedtuple also takes a "verbose" argument, but we don't care about that.
# rename will take any problematic field names and give them a new name.
# Like the other args, it's stored as a Variable, but we want just a bool.
if callargs["rename"]:
rename = abstract.get_atomic_python_constant(callargs["rename"])
else:
rename = False
return name_var, field_names, rename
def _getargs(self, node, args):
self._match_args(node, args)
# Normally we would use typing.NamedTuple.__new__ to match args to
# parameters, but we can't import typing.
# TODO(tsudol): Replace with typing.NamedTuple.__new__.
f = function.Signature.from_param_names("typing.NamedTuple",
["typename", "fields"])
callargs = {
arg_name: arg_var
for arg_name, arg_var, _ in f.iter_args(args)
}
# typing.NamedTuple doesn't support rename or verbose
name_var = callargs["typename"]
fields_var = callargs["fields"]
fields = abstract.get_atomic_python_constant(fields_var)
# The fields is a list of tuples, so we need to deeply unwrap them.
fields = [abstract.get_atomic_python_constant(t) for t in fields]
# We need the actual string for the field names and the AtomicAbstractValue
# for the field types.
names = []
types = []
for (name, typ) in fields:
names.append(abstract.get_atomic_python_constant(name))
types.append(abstract.get_atomic_value(typ))
return name_var, names, types
def make_class(self, node, f_locals):
f_locals = abstract.get_atomic_python_constant(f_locals)
# retrieve __qualname__ to get the name of class
name = f_locals["__qualname__"]
# retrieve __annotations__ to get the dict
# with key-value pair of (variable, type)
anno = f_locals.get("__annotations__", {})
if anno:
anno = abstract.get_atomic_value(anno)
# assemble the arguments that are compatible with NamedTupleFuncBuilder.call
field_list = []
defaults = []
for k, v in anno.items():
if k in f_locals:
defaults.append(f_locals.get(k))
# TODO(ahxun): check if the value matches the declared type
k = self.vm.convert.constant_to_var(k, node=node)
field_list.append(self.vm.convert.build_tuple(node, (k, v)))
anno = self.vm.convert.build_list(node, field_list)
posargs = (name, anno)
args = abstract.FunctionArgs(posargs=posargs)
node, cls_var = self.namedtuple.call(node, None, args)
cls_val = abstract.get_atomic_value(cls_var)
if not isinstance(cls_val, abstract.Unsolvable):
# set __new__.__defaults__
defaults = abstract.Tuple(tuple(defaults),
self.vm).to_variable(node)
node, new_attr = self.vm.attribute_handler.get_attribute(
node, cls_val, "__new__")
new_attr = abstract.get_atomic_value(new_attr)
node = self.vm.attribute_handler.set_attribute(
node, new_attr, "__defaults__", defaults)
# set the attribute without overriding special namedtuple attributes
node, fields = self.vm.attribute_handler.get_attribute(
node, cls_val, "_fields")
fields = abstract.get_atomic_python_constant(fields, tuple)
fields = [
abstract.get_atomic_python_constant(field, str)
for field in fields
]
for key in f_locals:
if key in self._prohibited:
self.vm.errorlog.not_writable(self.vm.frames, cls_val, key)
if key not in self._special and key not in fields:
node = self.vm.attribute_handler.set_attribute(
node, cls_val, key, f_locals[key])
return node, cls_var
def convert_function_annotations(self, node, raw_annotations):
if raw_annotations:
# {"i": int, "return": str} is stored as (int, str, ("i", "return"))
names = abstract.get_atomic_python_constant(raw_annotations[-1])
type_list = raw_annotations[:-1]
annotations_list = []
for name, t in zip(names, type_list):
name = abstract.get_atomic_python_constant(name)
t = self.convert_function_type_annotation(node, name, t)
annotations_list.append((name, t))
return self.convert_annotations_list(annotations_list)
else:
return {}, {}
def call(self, node, _, args):
try:
name_var, field_names, field_types = self._getargs(node, args)
except abstract.ConversionError:
return node, self.vm.convert.unsolvable.to_variable(node)
try:
name = abstract.get_atomic_python_constant(name_var)
except abstract.ConversionError:
return node, self.vm.convert.unsolvable.to_variable(node)
try:
field_names = self._validate_and_rename_args(
name, field_names, False)
except ValueError as e:
self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames,
utils.message(e))
return node, self.vm.convert.unsolvable.to_variable(node)
annots, late_annots = self.vm.annotations_util.convert_annotations_list(
moves.zip(field_names, field_types))
if late_annots:
# We currently don't support forward references. Report if we find any,
# then continue by using Unsolvable instead.
self.vm.errorlog.not_supported_yet(
self.vm.frames, "Forward references in typing.NamedTuple")
field_types = [
annots.get(field_name, self.vm.convert.unsolvable)
for field_name in field_names
]
cls_var = self._build_namedtuple(name, field_names, field_types, node)
self.vm.trace_classdef(cls_var)
return node, cls_var
def call(self, node, func, args):
args = args.simplify(node)
self._match_args(node, args, match_all_views=True)
# As long as the types match we do not really care about the actual
# class name. But, if we have a string literal value as the name arg,
# we will use it.
name_arg = args.namedargs.get(self._name_arg_name) or args.posargs[0]
try:
_ = abstract.get_atomic_python_constant(name_arg, str)
except abstract.ConversionError:
name_arg = self.vm.convert.constant_to_var(
"_NewType_Internal_Class_Name_%d_" % self.internal_name_counter)
type_arg = args.namedargs.get(self._type_arg_name) or args.posargs[1]
try:
type_value = abstract.get_atomic_value(type_arg)
except abstract.ConversionError:
# We need the type arg to be an atomic value. If not, we just
# silently return unsolvable.
return node, self.vm.convert.create_new_unsolvable(node)
value_arg_name = "val"
constructor = abstract.SimpleFunction(
name="__init__",
param_names=("self", value_arg_name),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={value_arg_name: type_value},
late_annotations={},
vm=self.vm,
).to_variable(node)
members = abstract.Dict(self.vm)
members.set_str_item(node, "__init__", constructor)
return node, self.vm.make_class(node, name_arg, (type_arg,),
members.to_variable(node), None)
def call(self, node, func, args):
if len(args.posargs) == 4:
self.match_args(node, args) # May raise FailedFunctionCall.
cls, name_var, bases_var, class_dict_var = args.posargs
try:
bases = list(abstract.get_atomic_python_constant(bases_var))
if not bases:
bases = [
self.vm.convert.object_type.to_variable(
self.vm.root_cfg_node)
]
variable = self.vm.make_class(node, name_var, bases,
class_dict_var, cls)
except abstract.ConversionError:
pass
else:
return node, variable
elif (args.posargs and self.vm.callself_stack
and args.posargs[-1].data == self.vm.callself_stack[-1].data):
# We're calling type(self) in an __init__ method. A common pattern for
# making a class non-instantiable is:
# class Foo:
# def __init__(self):
# if type(self) is Foo:
# raise ...
# If we were to return 'Foo', pytype would think that this constructor
# can never return. The correct return type is something like
# TypeVar(bound=Foo), but we can't introduce a type parameter that isn't
# bound to a class or function, so we'll go with Any.
self.match_args(node, args) # May raise FailedFunctionCall.
return node, self.vm.convert.unsolvable.to_variable(node)
return super(TypeNew, self).call(node, func, args)
def convert_function_annotations(self, node, raw_annotations):
if raw_annotations:
# {"i": int, "return": str} is stored as (int, str, ("i", "return"))
names = abstract.get_atomic_python_constant(raw_annotations[-1])
type_list = raw_annotations[:-1]
annotations_list = []
for name, t in zip(names, type_list):
name = abstract.get_atomic_python_constant(name)
visible = t.Data(node)
if len(visible) > 1:
self.vm.errorlog.ambiguous_annotation(
self.vm.frames, visible, name)
t = None
else:
t = visible[0]
annotations_list.append((name, t))
return self.convert_annotations_list(annotations_list)
else:
return {}, {}
def call(self, node, func, args):
if len(args.posargs) == 4:
self._match_args(node, args) # May raise FailedFunctionCall.
cls, name_var, bases_var, class_dict_var = args.posargs
try:
bases = list(abstract.get_atomic_python_constant(bases_var))
if not bases:
bases = [
self.vm.convert.object_type.to_variable(
self.vm.root_cfg_node)
]
variable = self.vm.make_class(node, name_var, bases,
class_dict_var, cls)
except abstract.ConversionError:
pass
else:
return node, variable
return super(TypeNew, self).call(node, func, args)
def call(self, node, *args, **kwargs):
"""Call typing.TypeVar()."""
if len(args) < 1:
self.vm.errorlog.invalid_typevar(self.vm.frame.current_opcode,
"Need name as first parameter")
return node, self.vm.convert.unsolvable.to_variable(node)
try:
typevar_name = abstract.get_atomic_python_constant(args[0])
except abstract.ConversionError:
self.vm.errorlog.invalid_typevar(self.vm.frame.current_opcode,
"Name must be a constant string")
return node, self.vm.convert.unsolvable.to_variable(node)
constraints = args[1:]
bound = kwargs.get("bound")
# TODO(kramm): These are variables. We should convert them to booleans.
covariant = kwargs.get("covariant")
contravariant = kwargs.get("contravariant")
typevar = abstract.TypeVariable(typevar_name, self.vm, constraints,
bound, covariant, contravariant)
self.vm.trace_typevar(typevar_name, typevar)
return node, typevar.to_variable(node)
def call(self, node, func, args):
if self.vm.PY3:
# In Python 3, the type of IO object returned depends on the mode.
self.match_args(node, args) # May raise FailedFunctionCall.
sig, = self.signatures
callargs = {
name: var
for name, var, _ in sig.signature.iter_args(args)
}
try:
if "mode" not in callargs:
io_type = "Text" # The default mode is 'r'.
else:
mode = abstract.get_atomic_python_constant(
callargs["mode"])
io_type = "Binary" if "b" in mode else "Text"
except abstract.ConversionError:
pass
else:
return node, self.vm.convert.constant_to_var(
abstract.AsInstance(
self.vm.lookup_builtin("typing.%sIO" % io_type)), {},
node)
return super(Open, self).call(node, func, args)
def getitem_slot(self, node, slice_var):
slice_tuple = abstract.get_atomic_python_constant(slice_var)
values = tuple(s.Data(node)[0] for s in slice_tuple)
new_union = Union(self.name, self.vm, node, self.elements + values)
return node, new_union.to_variable(node, "Union")
def call(self, node, _, args):
"""Creates a namedtuple class definition.
Performs the same argument checking as collections.namedtuple, e.g. making
sure field names don't start with _ or digits, making sure no keywords are
used for the typename or field names, and so on. Because the methods of the
class have to be changed to match the number and names of the fields, we
construct pytd.Function and pytd.Constant instances for each member of the
class. Finally, the pytd.Class is wrapped in an abstract.PyTDClass.
If incorrect arguments are passed, a subclass of abstract.FailedFunctionCall
will be raised. Other cases may raise abstract.ConversionError exceptions,
such as when the arguments are in unicode or any of the arguments have
multiple bindings, but these are caught and return Any. This also occurs if
an argument to namedtuple is invalid, e.g. a keyword is used as a field name
and rename is False.
Arguments:
node: the current CFG node
_: the func binding, ignored.
args: an abstract.FunctionArgs instance
Returns:
a tuple of the given CFG node and an abstract.PyTDClass instance (wrapped
in a Variable) representing the constructed namedtuple class.
If a abstract.ConversionError occurs or if field names are invalid, this
function returns Unsolvable (in a Variable) instead of a PyTDClass.
Raises:
abstract.FailedFunctionCall: Raised by _getargs if any of the arguments
do not match the function signature.
"""
# If we can't extract the arguments, we take the easy way out and return Any
try:
name_var, field_names, rename = self._getargs(node, args)
except abstract.ConversionError:
return node, self.vm.convert.unsolvable.to_variable(node)
# We need the bare name for a few things, so pull that out now.
# The same unicode issue can strike here, so again return Any.
try:
name = abstract.get_atomic_python_constant(name_var)
except abstract.ConversionError:
return node, self.vm.convert.unsolvable.to_variable(node)
# namedtuple does some checking and optionally renaming of field names,
# so we do too.
try:
field_names = self._validate_and_rename_args(name, field_names, rename)
except ValueError as e:
self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames, e.message)
return node, self.vm.convert.unsolvable.to_variable(node)
name = namedtuple_name(name, field_names)
ast = namedtuple_ast(name, field_names,
python_version=self.vm.python_version)
mapping = self._get_known_types_mapping()
# A truly well-formed pyi for the namedtuple will have references to the new
# namedtuple class itself for all `self` params and as the return type for
# methods like __new__, _replace and _make. In order to do that, we need a
# ClassType.
cls_type = pytd.ClassType(name)
mapping[name] = cls_type
cls = ast.Lookup(name).Visit(visitors.ReplaceTypes(mapping))
cls_type.cls = cls
# Make sure all NamedType nodes have been replaced by ClassType nodes with
# filled cls pointers.
cls.Visit(visitors.VerifyLookup())
# We can't build the PyTDClass directly, and instead must run it through
# convert.constant_to_value first, for caching.
instance = self.vm.convert.constant_to_value(cls, {}, self.vm.root_cfg_node)
self.vm.trace_namedtuple(instance)
return node, instance.to_variable(node)
def getitem_slot(self, node, slice_var): slice_tuple = abstract.get_atomic_python_constant(slice_var) values = tuple(s.Data(node)[0] for s in slice_tuple) new_union = Union(self.name, self.vm, self.elements + values) return node, new_union.to_variable(node, "Union")
