def from_pytd(cls, vm, name, sig):
"""Construct an abstract signature from a pytd signature."""
# TODO(kramm): templates
pytd_annotations = [
(p.name, p.type)
for p in sig.params + (sig.starargs, sig.starstarargs)
if p is not None
]
pytd_annotations.append(("return", sig.return_type))
return cls(
name=name,
param_names=tuple(p.name for p in sig.params if not p.kwonly),
varargs_name=None if sig.starargs is None else sig.starargs.name,
kwonly_params=set(p.name for p in sig.params if p.kwonly),
kwargs_name=None
if sig.starstarargs is None else sig.starstarargs.name,
defaults={
p.name:
vm.convert.constant_to_var(p.type,
subst=datatypes.AliasingDict(),
node=vm.root_cfg_node)
for p in sig.params if p.optional
},
annotations={
name:
vm.convert.constant_to_value(typ,
subst=datatypes.AliasingDict(),
node=vm.root_cfg_node)
for name, typ in pytd_annotations
},
late_annotations={},
postprocess_annotations=False,
)
def compute_subst(self, node, formal_args, arg_dict, view, alias_map=None):
"""Compute information about type parameters using one-way unification.
Given the arguments of a function call, try to find a substitution that
matches them against the specified formal parameters.
Args:
node: The current CFG node.
formal_args: An iterable of (name, value) pairs of formal arguments.
arg_dict: A map of strings to pytd.Bindings instances.
view: A mapping of Variable to Value.
alias_map: Optionally, a datatypes.UnionFind, which stores all the type
renaming information, mapping of type parameter name to its
representative.
Returns:
A tuple (subst, name), with "subst" the datatypes.HashableDict if we found
a working substition, None otherwise, and "name" the bad parameter in case
subst=None.
"""
if not arg_dict:
# A call with no arguments always succeeds.
assert not formal_args
return datatypes.AliasingDict(), None
subst = datatypes.AliasingDict()
if alias_map:
subst.uf = alias_map
self._set_error_subst(None)
for name, formal in formal_args:
actual = arg_dict[name]
subst = self._match_value_against_type(actual, formal, subst, node, view)
if subst is None:
formal = self.vm.annotations_util.sub_one_annotation(
node, formal, [self._error_subst or {}])
return None, function.BadParam(name=name, expected=formal)
return datatypes.HashableDict(subst), None
def convert_as_instance_attribute(self, name, instance):
"""Convert `name` as an instance attribute.
This method is used by attribute.py to lazily load attributes on instances
of this PyTDClass. Calling this method directly should be avoided. Doing so
will create multiple copies of the same attribute, leading to subtle bugs.
Args:
name: The attribute name.
instance: An instance of this PyTDClass.
Returns:
The converted attribute.
"""
if name not in self.pytd_cls:
return None
c = self.pytd_cls.Lookup(name)
if isinstance(c, pytd.Constant):
try:
self._convert_member(c)
except self.ctx.convert.TypeParameterError:
# Constant c cannot be converted without type parameter substitutions,
# so it must be an instance attribute.
subst = datatypes.AliasingDict()
for itm in self.pytd_cls.template:
subst[itm.full_name] = self.ctx.convert.constant_to_value(
itm.type_param, {}).instantiate(self.ctx.root_node,
container=instance)
return self._convert_member(c, subst)
def test_aliasing_dict_with_union_find(self):
d = datatypes.AliasingDict()
d["alias1"] = "1"
d["alias3"] = "2"
d.add_alias("alias1", "alias2")
d.add_alias("alias4", "alias3")
self.assertEqual(d["alias1"], "1")
self.assertEqual(d["alias2"], "1")
self.assertEqual(d["alias3"], "2")
self.assertEqual(d["alias4"], "2")
d.add_alias("alias1", "alias3", str.__add__)
self.assertEqual(d["alias1"], "12")
self.assertEqual(d["alias2"], "12")
self.assertEqual(d["alias3"], "12")
self.assertEqual(d["alias4"], "12")
d["alias5"] = "34"
d.add_alias("alias5", "alias6")
d.add_alias("alias6", "alias7")
d.add_alias("alias7", "alias8")
self.assertEqual(d["alias5"], "34")
self.assertEqual(d["alias6"], "34")
self.assertEqual(d["alias7"], "34")
self.assertEqual(d["alias8"], "34")
d.add_alias("alias1", "alias8", str.__add__)
for i in range(1, 9):
self.assertEqual(d["alias" + str(i)], "1234")
def name_to_value(self, name, subst=None, ast=None):
if ast is None:
pytd_cls = self.vm.lookup_builtin(name)
else:
pytd_cls = ast.Lookup(name)
subst = subst or datatypes.AliasingDict()
return self.constant_to_value(pytd_cls, subst, self.vm.root_cfg_node)
def testAliasingDictValueMove(self):
d = datatypes.AliasingDict()
v = self.prog.NewVariable()
d["alias"] = v
d.add_alias("alias", "name")
self.assertEqual(d["name"], v)
self.assertEqual(d["alias"], d["name"])
def testNonemptyAliasingDictRealiasing(self):
d = datatypes.AliasingDict()
d.add_alias("alias", "name")
d["name"] = "hello"
d["name2"] = "world"
self.assertRaises(AssertionError, lambda: d.add_alias("name2", "name"))
d.add_alias("alias", "name")
def testAliasingDictTransitive(self):
d = datatypes.AliasingDict()
d.add_alias("alias1", "name")
d.add_alias("alias2", "alias1")
d["name"] = self.prog.NewVariable()
self.assertEqual(1, len(d))
self.assertEqual(d["name"], d["alias1"])
self.assertEqual(d["alias1"], d["alias2"])
def bases(self):
convert = self.ctx.convert
return [
convert.constant_to_var(base,
subst=datatypes.AliasingDict(),
node=self.ctx.root_node)
for base in self.pytd_cls.bases
]
def compute_subst(self, node, formal_args, arg_dict, view, alias_map=None):
"""Compute information about type parameters using one-way unification.
Given the arguments of a function call, try to find a substitution that
matches them against the specified formal parameters.
Args:
node: The current CFG node.
formal_args: An iterable of (name, value) pairs of formal arguments.
arg_dict: A map of strings to pytd.Bindings instances.
view: A mapping of Variable to Value.
alias_map: Optionally, a datatypes.UnionFind, which stores all the type
renaming information, mapping of type parameter name to its
representative.
Returns:
A tuple (subst, name), with "subst" the datatypes.HashableDict if we found
a working substition, None otherwise, and "name" the bad parameter in case
subst=None.
"""
if not arg_dict:
# A call with no arguments always succeeds.
assert not formal_args
return datatypes.AliasingDict(), None
subst = datatypes.AliasingDict()
if alias_map:
subst.uf = alias_map
self._set_error_subst(None)
self_subst = None
for name, formal in formal_args:
actual = arg_dict[name]
subst = self._match_value_against_type(actual, formal, subst, node, view)
if subst is None:
formal = self.vm.annotations_util.sub_one_annotation(
node, formal, [self._error_subst or {}])
return None, function.BadParam(name=name, expected=formal)
if name == "self":
self_subst = subst
if self_subst:
# Type parameters matched from a 'self' arg are class parameters whose
# values have been declared by the user, e.g.:
# x = Container[int](__any_object__)
# We should keep the 'int' value rather than using Union[int, Unknown].
for name, value in self_subst.items():
if any(not isinstance(v, abstract.Empty) for v in value.data):
subst[name] = value
return datatypes.HashableDict(subst), None
def test_aliasing_dict_get(self):
d = datatypes.AliasingDict()
d["alias1"] = "1"
d.add_alias("alias1", "alias2")
self.assertEqual(d.get("alias1"), "1")
self.assertEqual(d.get("alias2"), "1")
self.assertEqual(d.get("alias3", "2"), "2")
self.assertIsNone(d.get("alias3"))
def __init__(self, name, pytd_sig, vm):
super(PyTDSignature, self).__init__(vm)
self.name = name
self.pytd_sig = pytd_sig
self.param_types = [
self.vm.convert.constant_to_value(
p.type, subst=datatypes.AliasingDict(), node=self.vm.root_cfg_node)
for p in self.pytd_sig.params]
self.signature = Signature.from_pytd(vm, name, pytd_sig)
def testAliasingDictTransitiveValueMove(self):
d = datatypes.AliasingDict()
d.add_alias("alias2", "name")
v = self.prog.NewVariable()
d["alias1"] = v
d.add_alias("alias1", "alias2")
self.assertEqual(d["name"], v)
self.assertEqual(d["alias2"], d["name"])
self.assertEqual(d["alias1"], d["alias2"])
def __init__(self, name, pytd_sig, ctx):
super().__init__(ctx)
self.name = name
self.pytd_sig = pytd_sig
self.param_types = [
self.ctx.convert.constant_to_value(
p.type, subst=datatypes.AliasingDict(), node=self.ctx.root_node)
for p in self.pytd_sig.params
]
self.signature = function.Signature.from_pytd(ctx, name, pytd_sig)
def test_nonempty_aliasing_dict_realiasing(self):
d = datatypes.AliasingDict()
d.add_alias("alias", "name")
d["name"] = "hello"
d["name2"] = "world"
self.assertEqual("hello", d["alias"])
self.assertEqual("hello", d["name"])
self.assertEqual("world", d["name2"])
d.add_alias("name", "name2", op=lambda x, y: x + " " + y)
self.assertEqual("hello world", d["name"])
self.assertEqual("hello world", d["name2"])
self.assertEqual("hello world", d["alias"])
def _match_parameterized_protocol(self, left_methods, other_type, subst,
node, view):
"""Checks whether left_methods is compatible with a parameterized protocol.
Args:
left_methods: A dictionary name -> method. method can either be a
Variable or a pytd.Function.
other_type: A formal type of type abstract.ParameterizedClass.
subst: The current type parameter assignment.
node: The current CFG node.
view: The current mapping of Variable to Value.
Returns:
A new type parameter assignment if the matching succeeded, None otherwise.
"""
new_substs = []
for name in other_type.protocol_methods:
abstract_method = other_type.get_method(name)
if name in left_methods:
matching_left_method = left_methods[name]
else:
return None
converter = self.vm.convert.pytd_convert
for signature in abstract_method.signatures:
callable_signature = converter.signature_to_callable(
signature.signature, self.vm)
if isinstance(callable_signature, abstract.CallableClass):
# Prevent the matcher from trying to enforce contravariance on 'self'.
callable_signature.formal_type_parameters[0] = (
self.vm.convert.unsolvable)
annotation_subst = datatypes.AliasingDict()
if isinstance(other_type.base_cls, mixin.Class):
annotation_subst.uf = (
other_type.base_cls.all_formal_type_parameters.uf)
for (param,
value) in other_type.get_formal_type_parameters().items():
annotation_subst[param] = (
self.vm.annotations_util.instantiate_for_sub(
node, value))
annotated_callable = self.vm.annotations_util.sub_one_annotation(
node, callable_signature, [annotation_subst])
if isinstance(matching_left_method, pytd.Function):
matching_left_method = self.vm.convert.constant_to_var(
matching_left_method)
for m in matching_left_method.data:
match_result = self._match_type_against_type(
m, annotated_callable, subst, node, view)
if match_result is None:
return None
else:
new_substs.append(match_result)
return self._merge_substs(subst, new_substs)
def test_aliasing_dict_realiasing(self):
d = datatypes.AliasingDict()
d.add_alias("alias1", "name")
d.add_alias("alias2", "name")
d.add_alias("alias1", "name")
d.add_alias("alias2", "alias1")
d.add_alias("alias1", "alias2")
# Check that the name, alias1, and alias2 still all refer to the same key
var = self.prog.NewVariable()
d["alias1"] = var
self.assertEqual(len(d), 1)
self.assertEqual(var, d["name"])
self.assertEqual(var, d["alias1"])
self.assertEqual(var, d["alias2"])
def __init__(self, vm, name, member_map, ast):
"""Initialize the overlay.
Args:
vm: Instance of vm.VirtualMachine.
name: A string containing the name of the underlying module.
member_map: Dict of str to abstract.AtomicAbstractValues that provide type
information not available in the underlying module.
ast: An pytd.TypeDeclUnit containing the AST for the underlying module.
Used to access type information for members of the module that are not
explicitly provided by the overlay.
"""
super().__init__(vm, name, member_map, ast)
self.real_module = vm.convert.constant_to_value(
ast, subst=datatypes.AliasingDict(), node=vm.root_cfg_node)
def set_defaults(self, defaults):
"""Set signature's default arguments. Requires rebuilding PyTD signature.
Args:
defaults: An iterable of function argument defaults.
Returns:
Self with an updated signature.
"""
defaults = list(defaults)
params = []
for param in reversed(self.pytd_sig.params):
if defaults:
defaults.pop() # Discard the default. Unless we want to update type?
params.append(pytd.Parameter(
name=param.name,
type=param.type,
kind=param.kind,
optional=True,
mutated_type=param.mutated_type
))
else:
params.append(pytd.Parameter(
name=param.name,
type=param.type,
kind=param.kind,
optional=False, # Reset any previously-set defaults
mutated_type=param.mutated_type
))
new_sig = pytd.Signature(
params=tuple(reversed(params)),
starargs=self.pytd_sig.starargs,
starstarargs=self.pytd_sig.starstarargs,
return_type=self.pytd_sig.return_type,
exceptions=self.pytd_sig.exceptions,
template=self.pytd_sig.template
)
# Now update self
self.pytd_sig = new_sig
self.param_types = [
self.ctx.convert.constant_to_value(
p.type, subst=datatypes.AliasingDict(), node=self.ctx.root_node)
for p in self.pytd_sig.params
]
self.signature = function.Signature.from_pytd(
self.ctx, self.name, self.pytd_sig)
return self
def from_pytd(cls, ctx, name, sig):
"""Construct an abstract signature from a pytd signature."""
pytd_annotations = [
(p.name, p.type)
for p in sig.params + (sig.starargs, sig.starstarargs)
if p is not None
]
pytd_annotations.append(("return", sig.return_type))
def param_to_var(p):
return ctx.convert.constant_to_var(p.type,
subst=datatypes.AliasingDict(),
node=ctx.root_node)
param_names = []
posonly_count = 0
kwonly_params = []
for p in sig.params:
if p.kind == pytd.ParameterKind.KWONLY:
kwonly_params.append(p.name)
continue
param_names.append(p.name)
posonly_count += p.kind == pytd.ParameterKind.POSONLY
return cls(
name=name,
param_names=tuple(param_names),
posonly_count=posonly_count,
varargs_name=None if sig.starargs is None else sig.starargs.name,
kwonly_params=tuple(kwonly_params),
kwargs_name=None
if sig.starstarargs is None else sig.starstarargs.name,
defaults={
p.name: param_to_var(p)
for p in sig.params if p.optional
},
annotations={
name:
ctx.convert.constant_to_value(typ,
subst=datatypes.AliasingDict(),
node=ctx.root_node)
for name, typ in pytd_annotations
},
postprocess_annotations=False,
)
def __init__(self, name, pytd_cls, ctx):
# Apply decorators first, in case they set any properties that later
# initialization code needs to read.
self.has_explicit_init = any(x.name == "__init__"
for x in pytd_cls.methods)
pytd_cls, decorated = decorate.process_class(pytd_cls)
self.pytd_cls = pytd_cls
super().__init__(name, ctx)
if decorate.has_decorator(pytd_cls,
("typing.final", "typing_extensions.final")):
self.final = True
# Keep track of the names of final methods and instance variables.
self.final_members = {}
mm = {}
for val in pytd_cls.constants:
if isinstance(val.type, pytd.Annotated):
mm[val.name] = val.Replace(type=val.type.base_type)
elif (isinstance(val.type, pytd.GenericType)
and val.type.base_type.name == "typing.Final"):
self.final_members[val.name] = val
mm[val.name] = val.Replace(type=val.type.parameters[0])
else:
mm[val.name] = val
for val in pytd_cls.methods:
mm[val.name] = val
if val.is_final:
self.final_members[val.name] = val
for val in pytd_cls.classes:
mm[val.name.rsplit(".", 1)[-1]] = val
if pytd_cls.metaclass is None:
metaclass = None
else:
metaclass = self.ctx.convert.constant_to_value(
pytd_cls.metaclass,
subst=datatypes.AliasingDict(),
node=self.ctx.root_node)
self.official_name = self.name
self.slots = pytd_cls.slots
mixin.LazyMembers.init_mixin(self, mm)
self.is_dynamic = self.compute_is_dynamic()
class_mixin.Class.init_mixin(self, metaclass)
if decorated:
self._populate_decorator_metadata()
def _convert_member(self, member, subst=None):
"""Convert a member as a variable. For lazy lookup."""
subst = subst or datatypes.AliasingDict()
node = self.ctx.root_node
if isinstance(member, pytd.Constant):
return self.ctx.convert.constant_to_var(
abstract_utils.AsInstance(member.type), subst, node)
elif isinstance(member, pytd.Function):
c = self.ctx.convert.constant_to_value(member,
subst=subst,
node=node)
c.parent = self
return c.to_variable(node)
elif isinstance(member, pytd.Class):
return self.ctx.convert.constant_to_var(member,
subst=subst,
node=node)
else:
raise AssertionError("Invalid class member %s" %
pytd_utils.Print(member))
def testAliasingDictRealiasing(self):
d = datatypes.AliasingDict()
d.add_alias("alias1", "name")
d.add_alias("alias2", "name")
self.assertRaises(AssertionError,
lambda: d.add_alias("name", "other_name"))
try:
d.add_alias("alias1", "other_name")
except datatypes.AliasingDictConflictError as e:
self.assertEqual(e.existing_name, "name")
else:
self.fail("AliasingDictConflictError not raised")
d.add_alias("alias1", "name")
d.add_alias("alias2", "alias1")
d.add_alias("alias1", "alias2")
# Check that the name, alias1, and alias2 still all refer to the same key
var = self.prog.NewVariable()
d["alias1"] = var
self.assertEqual(1, len(d))
self.assertEqual(var, d["name"])
self.assertEqual(var, d["alias1"])
self.assertEqual(var, d["alias2"])
def testAliasingDict(self):
d = datatypes.AliasingDict()
# To avoid surprising behavior, we require desired dict functionality to be
# explicitly overridden
with self.assertRaises(NotImplementedError):
d.viewitems()
d.add_alias("alias", "name")
self.assertNotIn("alias", d)
self.assertNotIn("name", d)
var1 = self.prog.NewVariable()
d["alias"] = var1
self.assertIn("name", d)
self.assertIn("alias", d)
self.assertEqual(var1, d["name"])
self.assertEqual(d["name"], d["alias"])
self.assertEqual(d["alias"], d.get("alias"))
self.assertEqual(d["name"], d.get("name"))
self.assertEqual(None, d.get("other_name"))
var2 = self.prog.NewVariable()
d["name"] = var2
self.assertEqual(var2, d["name"])
self.assertEqual(d["name"], d["alias"])
def call_slot(self, node, *args, **kwargs):
"""Implementation of CallableClass.__call__."""
if kwargs:
raise function.WrongKeywordArgs(
function.Signature.from_callable(self),
function.Args(posargs=args, namedargs=kwargs), self.ctx,
kwargs.keys())
if len(args) != self.num_args:
raise function.WrongArgCount(
function.Signature.from_callable(self),
function.Args(posargs=args), self.ctx)
formal_args = [(function.argname(i), self.formal_type_parameters[i])
for i in range(self.num_args)]
substs = [datatypes.AliasingDict()]
bad_param = None
for view in abstract_utils.get_views(args, node):
arg_dict = {
function.argname(i): view[args[i]]
for i in range(self.num_args)
}
subst, bad_param = self.ctx.matcher(node).compute_subst(
formal_args, arg_dict, view, None)
if subst is not None:
substs = [subst]
break
else:
if bad_param:
raise function.WrongArgTypes(
function.Signature.from_callable(self),
function.Args(posargs=args),
self.ctx,
bad_param=bad_param)
ret = self.ctx.annotation_utils.sub_one_annotation(
node, self.formal_type_parameters[abstract_utils.RET], substs)
node, retvar = self.ctx.vm.init_class(node, ret)
return node, retvar
def getitem_slot(self, node, slice_var):
"""Custom __getitem__ implementation."""
slice_content = abstract_utils.maybe_extract_tuple(slice_var)
params = self.ctx.annotation_utils.get_type_parameters(self)
num_params = len({x.name for x in params})
# Check that we are instantiating all the unbound type parameters
if num_params != len(slice_content):
self.ctx.errorlog.wrong_annotation_parameter_count(
self.ctx.vm.frames, self, [v.data[0] for v in slice_content],
num_params)
return node, self.ctx.new_unsolvable(node)
concrete = (var.data[0].instantiate(
node, container=abstract_utils.DUMMY_CONTAINER)
for var in slice_content)
subst = datatypes.AliasingDict()
for p in params:
for k in subst:
if k == p.name or k.endswith(f".{p.name}"):
subst.add_alias(p.full_name, k)
break
else:
subst[p.full_name] = next(concrete)
new = self.ctx.annotation_utils.sub_one_annotation(node, self, [subst])
return node, new.to_variable(node)
def param_to_var(p):
return vm.convert.constant_to_var(p.type,
subst=datatypes.AliasingDict(),
node=vm.root_cfg_node)
