def signature_to_callable(self, sig, vm):
"""Converts a function.Signature object into a callable object.
Args:
sig: The signature to convert.
vm: The vm instance.
Returns:
An abstract.Callable representing the signature, or an
abstract.ParameterizedClass if the signature has a variable number of
arguments.
"""
base_cls = vm.convert.function_type
ret = sig.annotations.get("return", vm.convert.unsolvable)
if self._detailed or (sig.mandatory_param_count()
== sig.maximum_param_count()):
# If self._detailed is false, we throw away the argument types if the
# function takes a variable number of arguments, which is correct for pyi
# generation but undesirable for, say, error message printing.
args = [
sig.annotations.get(name, vm.convert.unsolvable)
for name in sig.param_names
]
params = {
abstract.ARGS: abstract.merge_values(args, vm),
abstract.RET: ret
}
params.update(enumerate(args))
return abstract.Callable(base_cls, params, vm)
else:
# The only way to indicate a variable number of arguments in a Callable
# is to not specify argument types at all.
params = {abstract.ARGS: vm.convert.unsolvable, abstract.RET: ret}
return abstract.ParameterizedClass(base_cls, params, vm)
def _filter_and_merge_candidates(self, node, candidates):
"""Merge the given candidates into one variable, filtered by the node."""
ret = self.vm.program.NewVariable()
for candidate in candidates:
for binding in candidate.Bindings(node):
val = binding.data
if isinstance(val, abstract.TypeParameterInstance):
var = val.instance.type_parameters[val.name]
# If this type parameter has visible values, we add those to the
# return value. Otherwise, if it has constraints, we add those as an
# upper bound on the values. When all else fails, we add an empty
# value as a placeholder that can be passed around and converted to
# Any after analysis.
if var.bindings:
candidates.append(var)
elif val.param.constraints:
constraints = abstract.merge_values(
val.param.constraints, self.vm)
ret.PasteVariable(constraints.instantiate(node))
else:
ret.AddBinding(self.vm.convert.empty, [], node)
else:
sources = {binding}
ret.AddBinding(val, sources, node)
if ret.bindings:
return ret
else:
return None
def attribute_error(self, opcode, obj, attr_name):
assert obj.bindings
obj_values = abstract.merge_values(obj.data, obj.bindings[0].data.vm)
if isinstance(obj_values, abstract.Module):
obj_repr = "module %r" % obj_values.name
else:
obj_repr = self._print_as_actual_type(obj_values)
self.error(opcode, "No attribute %r on %s" % (attr_name, obj_repr))
def _get_value_info(self, inner, ends_with_ellipsis):
if not ends_with_ellipsis:
template = range(len(inner)) + [abstract.T]
inner += (abstract.merge_values(inner, self.vm), )
return template, inner, abstract.TupleClass
else:
return super(Tuple, self)._get_value_info(inner,
ends_with_ellipsis)
def _get_value_info(self, inner, ellipses):
if ellipses:
# An ellipsis may appear at the end of the parameter list as long as it is
# not the only parameter.
return super(Tuple, self)._get_value_info(
inner, ellipses, allowed_ellipses={len(inner) - 1} - {0})
else:
template = list(moves.range(len(inner))) + [abstract.T]
inner += (abstract.merge_values(inner, self.vm), )
return template, inner, abstract.TupleClass
def _get_value_info(self, inner, ends_with_ellipsis):
if isinstance(inner[0], list):
template = range(len(
inner[0])) + [t.name for t in self.base_cls.template]
combined_args = abstract.merge_values(inner[0], self.vm)
inner = tuple(inner[0]) + (combined_args, ) + inner[1:]
return template, inner, abstract.Callable
else:
return super(Callable,
self)._get_value_info(inner, ends_with_ellipsis)
def _attribute_error(self, stack, binding, attr_name):
obj_repr = self._print_as_actual_type(binding.data)
if len(binding.variable.bindings) > 1:
details = "In %s" % self._print_as_actual_type(
abstract.merge_values(binding.variable.data, binding.data.vm))
else:
details = None
self.error(stack,
"No attribute %r on %s" % (attr_name, obj_repr),
details=details)
def _get_value_info(self, inner, ellipses):
if isinstance(inner[0], list):
template = (list(moves.range(len(inner[0]))) +
[t.name for t in self.base_cls.template])
combined_args = abstract.merge_values(inner[0], self.vm)
inner = tuple(inner[0]) + (combined_args,) + inner[1:]
self.vm.errorlog.invalid_ellipses(self.vm.frames, ellipses, self.name)
return template, inner, abstract.Callable
else:
# An ellipsis may take the place of the ARGS list.
return super(Callable, self)._get_value_info(
inner, ellipses, allowed_ellipses={0})
def call(self, node, func, args):
if args.posargs:
try:
annot = self.vm.annotations_util.process_annotation_var(
args.posargs[0], "typing.cast", self.vm.frames, node)
except self.vm.annotations_util.LateAnnotationError:
self.vm.errorlog.invalid_annotation(
self.vm.frames,
abstract.merge_values(args.posargs[0].data, self.vm),
"Forward references not allowed in typing.cast.\n"
"Consider switching to a type comment.")
annot = self.vm.convert.create_new_unsolvable(node)
args = args.replace(posargs=(annot, ) + args.posargs[1:])
return super(Cast, self).call(node, func, args)
def merge_classes(self, node, instances):
"""Merge the classes of the given instances.
Args:
node: The current node.
instances: An iterable of instances.
Returns:
An abstract.AtomicAbstractValue created by merging the instances' classes.
"""
classes = set()
for v in instances:
cls = v.get_class()
if cls:
classes.add(cls)
return abstract.merge_values(classes, self.vm)
def _filter_var(self, node, var):
"""Filter the variable by the node.
Filters the variable data, including recursively expanded type parameter
instances, by visibility at the node. A type parameter instance needs to be
filtered at the moment of access because its value may change later.
Args:
node: The current node.
var: A variable to filter.
Returns:
The filtered variable.
"""
# First, check if we need to do any filtering at all. This method is
# heavily called, so creating the `ret` variable judiciously reduces the
# number of variables per pytype run by as much as 20%.
bindings = var.Bindings(node)
if not bindings:
return None
if len(bindings) == len(var.bindings) and not any(
isinstance(b.data, abstract.TypeParameterInstance) for b in bindings):
return var
ret = self.vm.program.NewVariable()
for binding in bindings:
val = binding.data
if isinstance(val, abstract.TypeParameterInstance):
var = val.instance.type_parameters[val.name]
# If this type parameter has visible values, we add those to the
# return value. Otherwise, if it has constraints, we add those as an
# upper bound on the values. When all else fails, we add an empty
# value as a placeholder that can be passed around and converted to
# Any after analysis.
var_bindings = var.Bindings(node)
if var_bindings:
bindings.extend(var_bindings)
elif val.param.constraints:
constraints = abstract.merge_values(val.param.constraints, self.vm)
ret.PasteVariable(constraints.instantiate(node))
else:
ret.AddBinding(self.vm.convert.empty, [], node)
else:
ret.AddBinding(val, {binding}, node)
if ret.bindings:
return ret
else:
return None
def not_writable(self, stack, obj, attr_name): obj_values = abstract.merge_values([obj], obj.vm) obj_repr = self._print_as_actual_type(obj_values) self.error(stack, "Can't assign attribute %r on %s" % (attr_name, obj_repr))
def attribute_error(self, stack, obj, attr_name):
assert obj.bindings
obj_values = abstract.merge_values(obj.data, obj.data[0].vm)
obj_repr = self._print_as_actual_type(obj_values)
self.error(stack, "No attribute %r on %s" % (attr_name, obj_repr))
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if isinstance(pyval, str):
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, long):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
if pyval is types.FunctionType:
classname = "typing.Callable"
else:
classname = "__builtin__." + pyval.__name__
try:
return self.name_to_value(classname, subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.TypeDeclUnit):
data = (pyval.constants + pyval.type_params + pyval.classes +
pyval.functions + pyval.aliases)
members = {val.name.rsplit(".")[-1]: val for val in data}
return abstract.Module(self.vm, pyval.name, members, pyval)
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.object":
return self.object_type
elif isinstance(pyval,
pytd.Class) and pyval.name == "types.ModuleType":
return self.module_type
elif isinstance(pyval, pytd.Class):
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.ExternalFunction):
module, _, name = pyval.name.partition(".")
assert module == "__builtin__", "PYTHONCODE allowed only in __builtin__"
return abstract.merge_values(
self.vm.frame.f_globals.members[name].data, self.vm)
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.nothing
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.Class):
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
instance.make_template_unsolvable(
cls.template, self.vm.root_cfg_node)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.GenericType):
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
assert len(cls.parameters) <= len(base_cls.template)
for formal, actual in zip(base_cls.template,
cls.parameters):
p = self.constant_to_var(abstract.AsInstance(actual),
subst, self.vm.root_cfg_node)
instance.initialize_type_parameter(
get_node(), formal.name, p)
return instance
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
assert isinstance(pyval.base_type, pytd.ClassType)
base_cls = self.constant_to_value(pyval.base_type.cls, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = range(len(pyval.parameters)) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = range(len(
pyval.args)) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in pyval.base_type.cls.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = utils.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def attribute_error(self, opcode, obj, attr_name):
assert obj.bindings
obj_values = self._print_as_actual_type(
abstract.merge_values(obj.data, obj.bindings[0].data.vm))
self.error(opcode, "No attribute %r on %s" % (attr_name, obj_values))