def from_pytd(cls, vm, name, sig): """Construct an abstract signature from a pytd signature.""" # TODO(kramm): templates pytd_annotations = [ (p.name, p.type) for p in sig.params + (sig.starargs, sig.starstarargs) if p is not None ] pytd_annotations.append(("return", sig.return_type)) return cls( name=name, param_names=tuple(p.name for p in sig.params if not p.kwonly), varargs_name=None if sig.starargs is None else sig.starargs.name, kwonly_params=set(p.name for p in sig.params if p.kwonly), kwargs_name=None if sig.starstarargs is None else sig.starstarargs.name, defaults={ p.name: vm.convert.constant_to_var(p.type, subst=datatypes.AliasingDict(), node=vm.root_cfg_node) for p in sig.params if p.optional }, annotations={ name: vm.convert.constant_to_value(typ, subst=datatypes.AliasingDict(), node=vm.root_cfg_node) for name, typ in pytd_annotations }, late_annotations={}, postprocess_annotations=False, )
def compute_subst(self, node, formal_args, arg_dict, view, alias_map=None): """Compute information about type parameters using one-way unification. Given the arguments of a function call, try to find a substitution that matches them against the specified formal parameters. Args: node: The current CFG node. formal_args: An iterable of (name, value) pairs of formal arguments. arg_dict: A map of strings to pytd.Bindings instances. view: A mapping of Variable to Value. alias_map: Optionally, a datatypes.UnionFind, which stores all the type renaming information, mapping of type parameter name to its representative. Returns: A tuple (subst, name), with "subst" the datatypes.HashableDict if we found a working substition, None otherwise, and "name" the bad parameter in case subst=None. """ if not arg_dict: # A call with no arguments always succeeds. assert not formal_args return datatypes.AliasingDict(), None subst = datatypes.AliasingDict() if alias_map: subst.uf = alias_map self._set_error_subst(None) for name, formal in formal_args: actual = arg_dict[name] subst = self._match_value_against_type(actual, formal, subst, node, view) if subst is None: formal = self.vm.annotations_util.sub_one_annotation( node, formal, [self._error_subst or {}]) return None, function.BadParam(name=name, expected=formal) return datatypes.HashableDict(subst), None
def convert_as_instance_attribute(self, name, instance): """Convert `name` as an instance attribute. This method is used by attribute.py to lazily load attributes on instances of this PyTDClass. Calling this method directly should be avoided. Doing so will create multiple copies of the same attribute, leading to subtle bugs. Args: name: The attribute name. instance: An instance of this PyTDClass. Returns: The converted attribute. """ if name not in self.pytd_cls: return None c = self.pytd_cls.Lookup(name) if isinstance(c, pytd.Constant): try: self._convert_member(c) except self.ctx.convert.TypeParameterError: # Constant c cannot be converted without type parameter substitutions, # so it must be an instance attribute. subst = datatypes.AliasingDict() for itm in self.pytd_cls.template: subst[itm.full_name] = self.ctx.convert.constant_to_value( itm.type_param, {}).instantiate(self.ctx.root_node, container=instance) return self._convert_member(c, subst)
def test_aliasing_dict_with_union_find(self): d = datatypes.AliasingDict() d["alias1"] = "1" d["alias3"] = "2" d.add_alias("alias1", "alias2") d.add_alias("alias4", "alias3") self.assertEqual(d["alias1"], "1") self.assertEqual(d["alias2"], "1") self.assertEqual(d["alias3"], "2") self.assertEqual(d["alias4"], "2") d.add_alias("alias1", "alias3", str.__add__) self.assertEqual(d["alias1"], "12") self.assertEqual(d["alias2"], "12") self.assertEqual(d["alias3"], "12") self.assertEqual(d["alias4"], "12") d["alias5"] = "34" d.add_alias("alias5", "alias6") d.add_alias("alias6", "alias7") d.add_alias("alias7", "alias8") self.assertEqual(d["alias5"], "34") self.assertEqual(d["alias6"], "34") self.assertEqual(d["alias7"], "34") self.assertEqual(d["alias8"], "34") d.add_alias("alias1", "alias8", str.__add__) for i in range(1, 9): self.assertEqual(d["alias" + str(i)], "1234")
def name_to_value(self, name, subst=None, ast=None): if ast is None: pytd_cls = self.vm.lookup_builtin(name) else: pytd_cls = ast.Lookup(name) subst = subst or datatypes.AliasingDict() return self.constant_to_value(pytd_cls, subst, self.vm.root_cfg_node)
def testAliasingDictValueMove(self): d = datatypes.AliasingDict() v = self.prog.NewVariable() d["alias"] = v d.add_alias("alias", "name") self.assertEqual(d["name"], v) self.assertEqual(d["alias"], d["name"])
def testNonemptyAliasingDictRealiasing(self): d = datatypes.AliasingDict() d.add_alias("alias", "name") d["name"] = "hello" d["name2"] = "world" self.assertRaises(AssertionError, lambda: d.add_alias("name2", "name")) d.add_alias("alias", "name")
def testAliasingDictTransitive(self): d = datatypes.AliasingDict() d.add_alias("alias1", "name") d.add_alias("alias2", "alias1") d["name"] = self.prog.NewVariable() self.assertEqual(1, len(d)) self.assertEqual(d["name"], d["alias1"]) self.assertEqual(d["alias1"], d["alias2"])
def bases(self): convert = self.ctx.convert return [ convert.constant_to_var(base, subst=datatypes.AliasingDict(), node=self.ctx.root_node) for base in self.pytd_cls.bases ]
def compute_subst(self, node, formal_args, arg_dict, view, alias_map=None): """Compute information about type parameters using one-way unification. Given the arguments of a function call, try to find a substitution that matches them against the specified formal parameters. Args: node: The current CFG node. formal_args: An iterable of (name, value) pairs of formal arguments. arg_dict: A map of strings to pytd.Bindings instances. view: A mapping of Variable to Value. alias_map: Optionally, a datatypes.UnionFind, which stores all the type renaming information, mapping of type parameter name to its representative. Returns: A tuple (subst, name), with "subst" the datatypes.HashableDict if we found a working substition, None otherwise, and "name" the bad parameter in case subst=None. """ if not arg_dict: # A call with no arguments always succeeds. assert not formal_args return datatypes.AliasingDict(), None subst = datatypes.AliasingDict() if alias_map: subst.uf = alias_map self._set_error_subst(None) self_subst = None for name, formal in formal_args: actual = arg_dict[name] subst = self._match_value_against_type(actual, formal, subst, node, view) if subst is None: formal = self.vm.annotations_util.sub_one_annotation( node, formal, [self._error_subst or {}]) return None, function.BadParam(name=name, expected=formal) if name == "self": self_subst = subst if self_subst: # Type parameters matched from a 'self' arg are class parameters whose # values have been declared by the user, e.g.: # x = Container[int](__any_object__) # We should keep the 'int' value rather than using Union[int, Unknown]. for name, value in self_subst.items(): if any(not isinstance(v, abstract.Empty) for v in value.data): subst[name] = value return datatypes.HashableDict(subst), None
def test_aliasing_dict_get(self): d = datatypes.AliasingDict() d["alias1"] = "1" d.add_alias("alias1", "alias2") self.assertEqual(d.get("alias1"), "1") self.assertEqual(d.get("alias2"), "1") self.assertEqual(d.get("alias3", "2"), "2") self.assertIsNone(d.get("alias3"))
def __init__(self, name, pytd_sig, vm): super(PyTDSignature, self).__init__(vm) self.name = name self.pytd_sig = pytd_sig self.param_types = [ self.vm.convert.constant_to_value( p.type, subst=datatypes.AliasingDict(), node=self.vm.root_cfg_node) for p in self.pytd_sig.params] self.signature = Signature.from_pytd(vm, name, pytd_sig)
def testAliasingDictTransitiveValueMove(self): d = datatypes.AliasingDict() d.add_alias("alias2", "name") v = self.prog.NewVariable() d["alias1"] = v d.add_alias("alias1", "alias2") self.assertEqual(d["name"], v) self.assertEqual(d["alias2"], d["name"]) self.assertEqual(d["alias1"], d["alias2"])
def __init__(self, name, pytd_sig, ctx): super().__init__(ctx) self.name = name self.pytd_sig = pytd_sig self.param_types = [ self.ctx.convert.constant_to_value( p.type, subst=datatypes.AliasingDict(), node=self.ctx.root_node) for p in self.pytd_sig.params ] self.signature = function.Signature.from_pytd(ctx, name, pytd_sig)
def test_nonempty_aliasing_dict_realiasing(self): d = datatypes.AliasingDict() d.add_alias("alias", "name") d["name"] = "hello" d["name2"] = "world" self.assertEqual("hello", d["alias"]) self.assertEqual("hello", d["name"]) self.assertEqual("world", d["name2"]) d.add_alias("name", "name2", op=lambda x, y: x + " " + y) self.assertEqual("hello world", d["name"]) self.assertEqual("hello world", d["name2"]) self.assertEqual("hello world", d["alias"])
def _match_parameterized_protocol(self, left_methods, other_type, subst, node, view): """Checks whether left_methods is compatible with a parameterized protocol. Args: left_methods: A dictionary name -> method. method can either be a Variable or a pytd.Function. other_type: A formal type of type abstract.ParameterizedClass. subst: The current type parameter assignment. node: The current CFG node. view: The current mapping of Variable to Value. Returns: A new type parameter assignment if the matching succeeded, None otherwise. """ new_substs = [] for name in other_type.protocol_methods: abstract_method = other_type.get_method(name) if name in left_methods: matching_left_method = left_methods[name] else: return None converter = self.vm.convert.pytd_convert for signature in abstract_method.signatures: callable_signature = converter.signature_to_callable( signature.signature, self.vm) if isinstance(callable_signature, abstract.CallableClass): # Prevent the matcher from trying to enforce contravariance on 'self'. callable_signature.formal_type_parameters[0] = ( self.vm.convert.unsolvable) annotation_subst = datatypes.AliasingDict() if isinstance(other_type.base_cls, mixin.Class): annotation_subst.uf = ( other_type.base_cls.all_formal_type_parameters.uf) for (param, value) in other_type.get_formal_type_parameters().items(): annotation_subst[param] = ( self.vm.annotations_util.instantiate_for_sub( node, value)) annotated_callable = self.vm.annotations_util.sub_one_annotation( node, callable_signature, [annotation_subst]) if isinstance(matching_left_method, pytd.Function): matching_left_method = self.vm.convert.constant_to_var( matching_left_method) for m in matching_left_method.data: match_result = self._match_type_against_type( m, annotated_callable, subst, node, view) if match_result is None: return None else: new_substs.append(match_result) return self._merge_substs(subst, new_substs)
def test_aliasing_dict_realiasing(self): d = datatypes.AliasingDict() d.add_alias("alias1", "name") d.add_alias("alias2", "name") d.add_alias("alias1", "name") d.add_alias("alias2", "alias1") d.add_alias("alias1", "alias2") # Check that the name, alias1, and alias2 still all refer to the same key var = self.prog.NewVariable() d["alias1"] = var self.assertEqual(len(d), 1) self.assertEqual(var, d["name"]) self.assertEqual(var, d["alias1"]) self.assertEqual(var, d["alias2"])
def __init__(self, vm, name, member_map, ast): """Initialize the overlay. Args: vm: Instance of vm.VirtualMachine. name: A string containing the name of the underlying module. member_map: Dict of str to abstract.AtomicAbstractValues that provide type information not available in the underlying module. ast: An pytd.TypeDeclUnit containing the AST for the underlying module. Used to access type information for members of the module that are not explicitly provided by the overlay. """ super().__init__(vm, name, member_map, ast) self.real_module = vm.convert.constant_to_value( ast, subst=datatypes.AliasingDict(), node=vm.root_cfg_node)
def set_defaults(self, defaults): """Set signature's default arguments. Requires rebuilding PyTD signature. Args: defaults: An iterable of function argument defaults. Returns: Self with an updated signature. """ defaults = list(defaults) params = [] for param in reversed(self.pytd_sig.params): if defaults: defaults.pop() # Discard the default. Unless we want to update type? params.append(pytd.Parameter( name=param.name, type=param.type, kind=param.kind, optional=True, mutated_type=param.mutated_type )) else: params.append(pytd.Parameter( name=param.name, type=param.type, kind=param.kind, optional=False, # Reset any previously-set defaults mutated_type=param.mutated_type )) new_sig = pytd.Signature( params=tuple(reversed(params)), starargs=self.pytd_sig.starargs, starstarargs=self.pytd_sig.starstarargs, return_type=self.pytd_sig.return_type, exceptions=self.pytd_sig.exceptions, template=self.pytd_sig.template ) # Now update self self.pytd_sig = new_sig self.param_types = [ self.ctx.convert.constant_to_value( p.type, subst=datatypes.AliasingDict(), node=self.ctx.root_node) for p in self.pytd_sig.params ] self.signature = function.Signature.from_pytd( self.ctx, self.name, self.pytd_sig) return self
def from_pytd(cls, ctx, name, sig): """Construct an abstract signature from a pytd signature.""" pytd_annotations = [ (p.name, p.type) for p in sig.params + (sig.starargs, sig.starstarargs) if p is not None ] pytd_annotations.append(("return", sig.return_type)) def param_to_var(p): return ctx.convert.constant_to_var(p.type, subst=datatypes.AliasingDict(), node=ctx.root_node) param_names = [] posonly_count = 0 kwonly_params = [] for p in sig.params: if p.kind == pytd.ParameterKind.KWONLY: kwonly_params.append(p.name) continue param_names.append(p.name) posonly_count += p.kind == pytd.ParameterKind.POSONLY return cls( name=name, param_names=tuple(param_names), posonly_count=posonly_count, varargs_name=None if sig.starargs is None else sig.starargs.name, kwonly_params=tuple(kwonly_params), kwargs_name=None if sig.starstarargs is None else sig.starstarargs.name, defaults={ p.name: param_to_var(p) for p in sig.params if p.optional }, annotations={ name: ctx.convert.constant_to_value(typ, subst=datatypes.AliasingDict(), node=ctx.root_node) for name, typ in pytd_annotations }, postprocess_annotations=False, )
def __init__(self, name, pytd_cls, ctx): # Apply decorators first, in case they set any properties that later # initialization code needs to read. self.has_explicit_init = any(x.name == "__init__" for x in pytd_cls.methods) pytd_cls, decorated = decorate.process_class(pytd_cls) self.pytd_cls = pytd_cls super().__init__(name, ctx) if decorate.has_decorator(pytd_cls, ("typing.final", "typing_extensions.final")): self.final = True # Keep track of the names of final methods and instance variables. self.final_members = {} mm = {} for val in pytd_cls.constants: if isinstance(val.type, pytd.Annotated): mm[val.name] = val.Replace(type=val.type.base_type) elif (isinstance(val.type, pytd.GenericType) and val.type.base_type.name == "typing.Final"): self.final_members[val.name] = val mm[val.name] = val.Replace(type=val.type.parameters[0]) else: mm[val.name] = val for val in pytd_cls.methods: mm[val.name] = val if val.is_final: self.final_members[val.name] = val for val in pytd_cls.classes: mm[val.name.rsplit(".", 1)[-1]] = val if pytd_cls.metaclass is None: metaclass = None else: metaclass = self.ctx.convert.constant_to_value( pytd_cls.metaclass, subst=datatypes.AliasingDict(), node=self.ctx.root_node) self.official_name = self.name self.slots = pytd_cls.slots mixin.LazyMembers.init_mixin(self, mm) self.is_dynamic = self.compute_is_dynamic() class_mixin.Class.init_mixin(self, metaclass) if decorated: self._populate_decorator_metadata()
def _convert_member(self, member, subst=None): """Convert a member as a variable. For lazy lookup.""" subst = subst or datatypes.AliasingDict() node = self.ctx.root_node if isinstance(member, pytd.Constant): return self.ctx.convert.constant_to_var( abstract_utils.AsInstance(member.type), subst, node) elif isinstance(member, pytd.Function): c = self.ctx.convert.constant_to_value(member, subst=subst, node=node) c.parent = self return c.to_variable(node) elif isinstance(member, pytd.Class): return self.ctx.convert.constant_to_var(member, subst=subst, node=node) else: raise AssertionError("Invalid class member %s" % pytd_utils.Print(member))
def testAliasingDictRealiasing(self): d = datatypes.AliasingDict() d.add_alias("alias1", "name") d.add_alias("alias2", "name") self.assertRaises(AssertionError, lambda: d.add_alias("name", "other_name")) try: d.add_alias("alias1", "other_name") except datatypes.AliasingDictConflictError as e: self.assertEqual(e.existing_name, "name") else: self.fail("AliasingDictConflictError not raised") d.add_alias("alias1", "name") d.add_alias("alias2", "alias1") d.add_alias("alias1", "alias2") # Check that the name, alias1, and alias2 still all refer to the same key var = self.prog.NewVariable() d["alias1"] = var self.assertEqual(1, len(d)) self.assertEqual(var, d["name"]) self.assertEqual(var, d["alias1"]) self.assertEqual(var, d["alias2"])
def testAliasingDict(self): d = datatypes.AliasingDict() # To avoid surprising behavior, we require desired dict functionality to be # explicitly overridden with self.assertRaises(NotImplementedError): d.viewitems() d.add_alias("alias", "name") self.assertNotIn("alias", d) self.assertNotIn("name", d) var1 = self.prog.NewVariable() d["alias"] = var1 self.assertIn("name", d) self.assertIn("alias", d) self.assertEqual(var1, d["name"]) self.assertEqual(d["name"], d["alias"]) self.assertEqual(d["alias"], d.get("alias")) self.assertEqual(d["name"], d.get("name")) self.assertEqual(None, d.get("other_name")) var2 = self.prog.NewVariable() d["name"] = var2 self.assertEqual(var2, d["name"]) self.assertEqual(d["name"], d["alias"])
def call_slot(self, node, *args, **kwargs): """Implementation of CallableClass.__call__.""" if kwargs: raise function.WrongKeywordArgs( function.Signature.from_callable(self), function.Args(posargs=args, namedargs=kwargs), self.ctx, kwargs.keys()) if len(args) != self.num_args: raise function.WrongArgCount( function.Signature.from_callable(self), function.Args(posargs=args), self.ctx) formal_args = [(function.argname(i), self.formal_type_parameters[i]) for i in range(self.num_args)] substs = [datatypes.AliasingDict()] bad_param = None for view in abstract_utils.get_views(args, node): arg_dict = { function.argname(i): view[args[i]] for i in range(self.num_args) } subst, bad_param = self.ctx.matcher(node).compute_subst( formal_args, arg_dict, view, None) if subst is not None: substs = [subst] break else: if bad_param: raise function.WrongArgTypes( function.Signature.from_callable(self), function.Args(posargs=args), self.ctx, bad_param=bad_param) ret = self.ctx.annotation_utils.sub_one_annotation( node, self.formal_type_parameters[abstract_utils.RET], substs) node, retvar = self.ctx.vm.init_class(node, ret) return node, retvar
def getitem_slot(self, node, slice_var): """Custom __getitem__ implementation.""" slice_content = abstract_utils.maybe_extract_tuple(slice_var) params = self.ctx.annotation_utils.get_type_parameters(self) num_params = len({x.name for x in params}) # Check that we are instantiating all the unbound type parameters if num_params != len(slice_content): self.ctx.errorlog.wrong_annotation_parameter_count( self.ctx.vm.frames, self, [v.data[0] for v in slice_content], num_params) return node, self.ctx.new_unsolvable(node) concrete = (var.data[0].instantiate( node, container=abstract_utils.DUMMY_CONTAINER) for var in slice_content) subst = datatypes.AliasingDict() for p in params: for k in subst: if k == p.name or k.endswith(f".{p.name}"): subst.add_alias(p.full_name, k) break else: subst[p.full_name] = next(concrete) new = self.ctx.annotation_utils.sub_one_annotation(node, self, [subst]) return node, new.to_variable(node)
def param_to_var(p): return vm.convert.constant_to_var(p.type, subst=datatypes.AliasingDict(), node=vm.root_cfg_node)