def unsupported_operands(self, opcode, node, operation, var1, var2):
left = pytd_utils.JoinTypes(t.to_type(node) for t in var1.data)
right = pytd_utils.JoinTypes(t.to_type(node) for t in var2.data)
# TODO(kramm): Display things like '__add__' as '+'
self.error(
opcode, "unsupported operand type(s) for %s: %r and %r" %
(operation, pytd.Print(left), pytd.Print(right)))
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, abstract.Nothing)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if (v.name in v.instance.type_parameters and
v.instance.type_parameters[v.name].bindings):
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, view)
for p in v.instance.type_parameters[v.name].data)
else:
# The type parameter was never initialized
return pytd.AnythingType()
elif isinstance(v, (abstract.Function, abstract.IsInstance,
abstract.BoundFunction)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("__builtin__.type"),
parameters=(param,))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
classvalues = self._get_values(node, v.cls, view)
cls_types = []
for cls in classvalues:
cls_types.append(self.value_instance_to_pytd_type(
node, cls, v, seen=seen, view=view))
ret = pytd_utils.JoinTypes(cls_types)
visitors.InPlaceFillInClasses(ret, v.vm.loader.builtins)
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(tuple(self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, abstract.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
else:
raise NotImplementedError(v.__class__.__name__)
def testJoinNothingType(self):
"""Test that JoinTypes() removes or collapses 'nothing'."""
a = pytd.NamedType("a")
nothing = pytd.NothingType()
self.assertEquals(utils.JoinTypes([a, nothing]), a)
self.assertEquals(utils.JoinTypes([nothing]), nothing)
self.assertEquals(utils.JoinTypes([nothing, nothing]), nothing)
def VisitFunction(self, f):
"""Shrink a function, by factorizing cartesian products of arguments.
Greedily groups signatures, looking at the arguments from left to right.
This algorithm is *not* optimal. But it does the right thing for the
typical cases.
Arguments:
f: An instance of pytd.Function. If this function has more
than one signature, we will try to combine some of these signatures by
introducing union types.
Returns:
A new, potentially optimized, instance of pytd.Function.
"""
max_argument_count = max(len(s.params) for s in f.signatures)
signatures = f.signatures
for i in xrange(max_argument_count):
new_sigs = []
for sig, types in self._GroupByOmittedArg(signatures, i):
if types:
# One or more options for argument <i>:
new_params = list(sig.params)
new_params[i] = pytd.Parameter(sig.params[i].name,
utils.JoinTypes(types))
sig = sig.Replace(params=tuple(new_params))
new_sigs.append(sig)
else:
# Signature doesn't have argument <i>, so we store the original:
new_sigs.append(sig)
signatures = new_sigs
return f.Replace(signatures=tuple(signatures))
def _check_function(self, pytd_function, f, node, skip_self=False):
"""Check that a function or method is compatible with its PYTD."""
for sig in pytd_function.signatures:
args = [self._create_call_arg(p.name, p.type, node)
for p in sig.params[(1 if skip_self else 0):]]
nominal_return = self.convert.convert_constant_to_value(
"ret", sig.return_type, subst={}, node=self.root_cfg_node)
for val in f.bindings:
fvar = val.AssignToNewVariable("f", node)
_, retvar = self.call_function_in_frame(
node, fvar, args, {}, None, None)
if retvar.bindings:
bad = abstract.bad_matches(retvar, nominal_return, node)
if bad:
if isinstance(val.data, (abstract.InterpreterFunction,
abstract.BoundInterpreterFunction)):
combined = pytd_utils.JoinTypes([t.data.to_type(node)
for t in bad])
self.errorlog.bad_return_type(
val.data.get_first_opcode(), pytd_function,
combined, sig.return_type)
else:
log.error("%s is not a function?", val.data.name)
else:
log.error("Couldn't call %s", pytd_function.name)
def _ReplaceByOuterIfNecessary(self, item, substitutions):
"""Potentially replace a function type param with a class type param.
Args:
item: A pytd.TemplateItem
substitutions: A dictionary to update with what we replaced.
Returns:
Either [item] or [].
"""
containing_union = self._AllContaining(item.type_param)
if not containing_union:
return [item]
class_type_parameters = [
type_param for type_param in containing_union
if self.IsClassTypeParameter(type_param)
]
if class_type_parameters:
substitutions[item.type_param] = utils.JoinTypes(
class_type_parameters)
return []
else:
# It's a function type parameter that appears in a union with other
# function type parameters.
# TODO(kramm): We could merge those, too.
return [item]
def _call_traces_to_function(call_traces, name_transform=lambda x: x):
funcs = collections.defaultdict(pytd_utils.OrderedSet)
for node, funcvar, args, kws, retvar in call_traces:
if isinstance(funcvar.data, abstract.BoundFunction):
func = funcvar.data.underlying.signatures[0]
else:
func = funcvar.data.signatures[0]
arg_names = func.get_positional_names()
arg_types = (a.data.to_type(node)
for a in func.get_bound_arguments() + list(args))
ret = pytd_utils.JoinTypes(t.to_type(node) for t in retvar.data)
# TODO(kramm): Record these:
starargs = None
starstarargs = None
funcs[funcvar.data.name].add(pytd.Signature(
tuple(pytd.Parameter(n, t, False, False, None)
for n, t in zip(arg_names, arg_types)) +
tuple(pytd.Parameter(name, a.data.to_type(node), False, False, None)
for name, a in kws),
starargs, starstarargs,
ret, exceptions=(), template=()))
functions = []
for name, signatures in funcs.items():
functions.append(pytd.Function(name_transform(name), tuple(signatures),
pytd.METHOD))
return functions
def pytd_for_types(self, defs, ignore):
for name, var in defs.items():
abstract.variable_set_official_name(var, name)
data = []
for name, var in defs.items():
if name in output.TOP_LEVEL_IGNORE or name in ignore:
continue
options = var.FilteredData(self.exitpoint)
if (len(options) > 1 and not
all(isinstance(o, (abstract.Function, abstract.BoundFunction))
for o in options)):
# It's ambiguous whether this is a type, a function or something
# else, so encode it as a constant.
combined_types = pytd_utils.JoinTypes(t.to_type(self.exitpoint)
for t in options)
data.append(pytd.Constant(name, combined_types))
else:
for option in options:
if hasattr(option, "to_pytd_def"):
d = option.to_pytd_def(self.exitpoint, name) # Deep definition
else:
d = option.to_type(self.exitpoint) # Type only
if isinstance(d, pytd.TYPE):
data.append(pytd.Constant(name, d))
else:
data.append(d)
return pytd_utils.WrapTypeDeclUnit("inferred", data)
def _check_return(self, node, actual, formal):
bad = self.matcher.bad_matches(actual, formal, node)
if bad:
combined = pytd_utils.JoinTypes(
view[actual].data.to_type(node, view=view) for view in bad)
self.errorlog.bad_return_type(self.frames, combined,
formal.get_instance_type(node))
def _call_traces_to_function(call_traces, name_transform=lambda x: x):
funcs = collections.defaultdict(pytd_utils.OrderedSet)
for node, func, sigs, args, kws, retvar in call_traces:
# The lengths may be different in the presence of optional and kw args.
arg_names = max((sig.get_positional_names() for sig in sigs),
key=len)
for i in range(len(arg_names)):
if not isinstance(func.data, abstract.BoundFunction) or i > 0:
arg_names[i] = function.argname(i)
arg_types = (a.data.to_type(node) for a in args)
ret = pytd_utils.JoinTypes(t.to_type(node) for t in retvar.data)
# TODO(kramm): Record these:
starargs = None
starstarargs = None
funcs[func.data.name].add(
pytd.Signature(tuple(
pytd.Parameter(n, t, False, False, None)
for n, t in zip(arg_names, arg_types)) + tuple(
pytd.Parameter(name, a.data.to_type(node), False,
False, None) for name, a in kws),
starargs,
starstarargs,
ret,
exceptions=(),
template=()))
functions = []
for name, signatures in funcs.items():
functions.append(
pytd.Function(name_transform(name), tuple(signatures),
pytd.METHOD))
return functions
def pytd_for_types(self, defs):
data = []
for name, var in defs.items():
if name in output.TOP_LEVEL_IGNORE or self._is_builtin(
name, var.data):
continue
options = var.FilteredData(self.exitpoint)
if (len(options) > 1 and not all(
isinstance(o, (abstract.Function, abstract.BoundFunction))
for o in options)):
# It's ambiguous whether this is a type, a function or something
# else, so encode it as a constant.
combined_types = pytd_utils.JoinTypes(
t.to_type(self.exitpoint) for t in options)
data.append(pytd.Constant(name, combined_types))
elif options:
for option in options:
try:
d = option.to_pytd_def(self.exitpoint,
name) # Deep definition
except NotImplementedError:
d = option.to_type(self.exitpoint) # Type only
if isinstance(d, pytd.NothingType):
assert isinstance(option, abstract.Empty)
d = pytd.AnythingType()
if isinstance(d, pytd.TYPE) and not isinstance(
d, pytd.TypeParameter):
data.append(pytd.Constant(name, d))
else:
data.append(d)
else:
log.error("No visible options for " + name)
data.append(pytd.Constant(name, pytd.AnythingType()))
return pytd_utils.WrapTypeDeclUnit("inferred", data)
def VisitUnionType(self, union):
"""Given a union type, try to find a simplification by using superclasses.
This is a lossy optimization that tries to map a list of types to a common
base type. For example, int and bool are both base classes of int, so it
would convert "int or bool" to "int".
Arguments:
union: A union type.
Returns:
A simplified type, if available.
"""
intersection = self._Expand(union.type_list[0])
for t in union.type_list[1:]:
intersection.intersection_update(self._Expand(t))
# Remove "redundant" superclasses, by removing everything from the tree
# that's not a leaf. I.e., we don't need "object" if we have more
# specialized types.
new_type_list = tuple(cls for cls in intersection
if not self._HasSubClassInSet(cls, intersection))
return utils.JoinTypes(new_type_list)
def assertOnlyHasReturnType(self, func, t):
"""Test that a given return type is the only one."""
ret = pytd_utils.JoinTypes(sig.return_type
for sig in func.signatures)
self.assertEquals(t, ret,
"Return type %r != %r" % (pytd.Print(t),
pytd.Print(ret)))
def _value_to_parameter_types(self, node, v, instance, template, seen, view):
"""Get PyTD types for the parameters of an instance of an abstract value."""
if self._is_tuple(v, instance):
assert len(template) == 1 and template[0] == abstract.T, template
if isinstance(v, abstract.TupleClass):
template = range(len(v.type_parameters) - 1)
else:
template = range(len(instance.pyval))
if instance is None and isinstance(v, abstract.ParameterizedClass):
return [self.value_instance_to_pytd_type(
node, v.type_parameters[t], None, seen, view) for t in template]
elif isinstance(instance, abstract.SimpleAbstractValue):
type_arguments = []
for t in template:
if isinstance(instance, abstract.Tuple):
param_values = self._get_values(node, instance.pyval[t], view)
elif t in instance.type_parameters:
param_values = self._get_values(
node, instance.type_parameters[t], view)
else:
param_values = [v.vm.convert.unsolvable]
type_arguments.append(pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, view) for p in param_values))
return type_arguments
else:
return [pytd.AnythingType() for _ in template]
def get_instance_type(self, node, instance=None, seen=None):
if self.inner:
t = pytd_utils.JoinTypes([i.get_instance_type(node, seen=seen)
for i in self.inner.data])
return pytd.GenericType(pytd.NamedType(self.pytd_name), (t,))
else:
return pytd.NamedType(self.pytd_name)
def _check_return(self, node, actual, formal):
bad = abstract.bad_matches(actual, formal, node)
if bad:
combined = pytd_utils.JoinTypes([t.data.to_type(node) for t in bad])
self.errorlog.bad_return_type(
self.frame.current_opcode, self,
combined, formal.get_instance_type(node)
)
def VisitUnionType(self, union):
"""Push unions down into containers.
This collects similar container types in unions and merges them into
single instances with the union type pushed down to the element_type level.
Arguments:
union: A pytd.Union instance. Might appear in a parameter, a return type,
a constant type, etc.
Returns:
A simplified pytd.Union.
"""
if not any(isinstance(t, pytd.GenericType) for t in union.type_list):
# Optimization: If we're not going to change anything, return original.
return union
union = utils.JoinTypes(union.type_list) # flatten
if not isinstance(union, pytd.UnionType):
union = pytd.UnionType((union, ))
collect = {}
has_redundant_base_types = False
for t in union.type_list:
if isinstance(t, pytd.GenericType):
if t.base_type in collect:
has_redundant_base_types = True
collect[t.base_type] = tuple(
utils.JoinTypes([p1, p2])
for p1, p2 in zip(collect[t.base_type], t.parameters))
else:
collect[t.base_type] = t.parameters
if not has_redundant_base_types:
return union
result = pytd.NothingType()
done = set()
for t in union.type_list:
if isinstance(t, pytd.GenericType):
if t.base_type in done:
continue # already added
parameters = collect[t.base_type]
add = t.Replace(parameters=tuple(
p.Visit(CombineContainers()) for p in parameters))
done.add(t.base_type)
else:
add = t
result = utils.JoinTypes([result, add])
return result
def convert_string_type_list(types_as_string, unknown, mapping,
global_lookup, depth=0):
"""Like convert_string_type, but operate on a list."""
if not types_as_string or booleq.Solver.ANY_VALUE in types_as_string:
# If we didn't find a solution for a type (the list of matches is empty)
# then report it as "?", not as "nothing", because the latter is confusing.
return pytd.AnythingType()
return pytd_utils.JoinTypes(convert_string_type(type_as_string, unknown,
mapping, global_lookup, depth)
for type_as_string in types_as_string)
def testJoinTypes(self):
"""Test that JoinTypes() does recursive flattening."""
n1, n2, n3, n4, n5, n6 = [pytd.NamedType("n%d" % i) for i in xrange(6)]
# n1 or (n2 or (n3))
nested1 = pytd.UnionType((n1, pytd.UnionType((n2, pytd.UnionType((n3,))))))
# ((n4) or n5) or n6
nested2 = pytd.UnionType((pytd.UnionType((pytd.UnionType((n4,)), n5)), n6))
joined = utils.JoinTypes([nested1, nested2])
self.assertEquals(joined.type_list,
(n1, n2, n3, n4, n5, n6))
def VisitUnionType(self, union):
c = collections.Counter()
for t in set(union.type_list):
# TODO(rechen): How can we make this work with GenericType?
if isinstance(t, pytd.GENERIC_BASE_TYPE):
c += collections.Counter(self.hierarchy.ExpandSubClasses(str(t)))
# Below, c[str[t]] can be zero - that's the default for non-existent items
# in collections.Counter. It'll happen for types that are not
# instances of GENERIC_BASE_TYPE, like container types.
new_type_list = [t for t in union.type_list if c[str(t)] <= 1]
return utils.JoinTypes(new_type_list)
def assertHasReturnType(self, func, t):
"""Test that a given return type is present. Ignore extras."""
ret = pytd_utils.JoinTypes(sig.return_type for sig in func.signatures)
if isinstance(ret, pytd.UnionType):
self.assertIn(
t, ret.type_list, "Return type %r not found in %r" %
(pytd.Print(t), pytd.Print(ret)))
else:
self.assertEqual(
t, ret,
"Return type %r != %r" % (pytd.Print(ret), pytd.Print(t)))
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
# class-level attributes
for name, member in v.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(v.vm.exitpoint):
if isinstance(value, abstract.Function):
val = self.value_to_pytd_def(node, value, name)
if isinstance(val, pytd.Function):
methods[name] = val
elif isinstance(v, pytd.TYPE):
constants[name].add_type(val)
else:
raise AssertionError(str(type(val)))
else:
constants[name].add_type(value.to_type(node))
# instance-level attributes
for instance in v.instances:
for name, member in instance.members.items():
if name not in CLASS_LEVEL_IGNORE:
for value in member.FilteredData(v.vm.exitpoint):
constants[name].add_type(value.to_type(node))
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
bases = [pytd_utils.JoinTypes(b.get_instance_type(node)
for b in basevar.data)
for basevar in v.bases()
if basevar is not v.vm.convert.oldstyleclass_type]
constants = [pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
return pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
template=())
def _call_traces_to_function(call_traces, prefix=""):
funcs = collections.defaultdict(pytd_utils.OrderedSet)
for funcvar, args, kws, retvar in call_traces:
if isinstance(funcvar.data, abstract.BoundFunction):
func = funcvar.data.underlying.signatures[0]
else:
func = funcvar.data.signatures[0]
arg_names = func.get_parameter_names()
arg_types = (a.data.to_type()
for a in func.get_bound_arguments() + list(args))
ret = pytd_utils.JoinTypes(t.to_type() for t in retvar.data)
funcs[funcvar.data.name].add(pytd.Signature(
tuple(pytd.Parameter(n, t)
for n, t in zip(arg_names, arg_types)) +
tuple(pytd.Parameter(name, a.data.to_type())
for name, a in kws),
ret, has_optional=False, exceptions=(), template=()))
functions = []
for name, signatures in funcs.items():
functions.append(pytd.Function(prefix + name, tuple(signatures)))
return functions
def VisitFunction(self, f):
"""Merge signatures of a function.
This groups signatures by arguments and then for each group creates a
single signature that joins the return values / exceptions using "or".
Arguments:
f: A pytd.Function instance
Returns:
Function with simplified / combined signatures.
"""
groups = self._GroupByArguments(f.signatures)
new_signatures = []
for stripped_signature, ret_exc in groups.items():
ret = utils.JoinTypes(ret_exc.return_types)
exc = tuple(ret_exc.exceptions)
new_signatures.append(
stripped_signature.Replace(return_type=ret, exceptions=exc))
return f.Replace(signatures=tuple(new_signatures))
def base_class_error(self, opcode, node, base_var):
pytd_type = pytd_utils.JoinTypes(
t.get_instance_type(node) for t in base_var.data)
self.error(opcode, "Invalid base class: %s" % pytd.Print(pytd_type))
def VisitUnionType(self, union):
return utils.JoinTypes(union.type_list)
def testJoinAnythingTypes(self):
"""Test that JoinTypes() simplifies unions containing '?'."""
types = [pytd.AnythingType(), pytd.NamedType("a")]
self.assertIsInstance(utils.JoinTypes(types), pytd.AnythingType)
def testJoinEmptyTypesToNothing(self):
"""Test that JoinTypes() simplifies empty unions to 'nothing'."""
self.assertIsInstance(utils.JoinTypes([]), pytd.NothingType)
def VisitMutableParameter(self, p):
return pytd.Parameter(p.name, utils.JoinTypes([p.type, p.new_type]))
def testJoinSingleType(self):
"""Test that JoinTypes() returns single types as-is."""
a = pytd.NamedType("a")
self.assertEquals(utils.JoinTypes([a]), a)
self.assertEquals(utils.JoinTypes([a, a]), a)