def test_getitem__abstract_index(self): t = abstract.Tuple((self._var, ), self._vm) index = self._vm.convert.build_int(self._node) node, var = t.cls.getitem_slot(self._node, index) self.assertIs(node, self._node) self.assertIs(abstract_utils.get_atomic_value(var), abstract_utils.get_atomic_value(self._var))
def make_class(self, node, f_locals): # If BuildClass.call() hits max depth, f_locals will be [unsolvable] # Since we don't support defining NamedTuple subclasses in a nested scope # anyway, we can just return unsolvable here to prevent a crash, and let the # invalid namedtuple error get raised later. if f_locals.data[0].isinstance_Unsolvable(): return node, self.vm.new_unsolvable(node) f_locals = abstract_utils.get_atomic_python_constant(f_locals) # retrieve __qualname__ to get the name of class name = f_locals["__qualname__"] # assemble the arguments that are compatible with NamedTupleFuncBuilder.call field_list = [] defaults = [] cls_locals = classgen.get_class_locals( abstract_utils.get_atomic_python_constant(name), allow_methods=True, ordering=classgen.Ordering.FIRST_ANNOTATE, vm=self.vm) for k, local in cls_locals.items(): assert local.typ if k in f_locals: defaults.append(f_locals[k]) k = self.vm.convert.constant_to_var(k, node=node) field_list.append(self.vm.convert.build_tuple( node, (k, local.typ))) anno = self.vm.convert.build_list(node, field_list) posargs = (name, anno) args = function.Args(posargs=posargs) node, cls_var = self.namedtuple.call(node, None, args) cls_val = abstract_utils.get_atomic_value(cls_var) if not isinstance(cls_val, abstract.Unsolvable): # set __new__.__defaults__ defaults = abstract.Tuple(tuple(defaults), self.vm).to_variable(node) node, new_attr = self.vm.attribute_handler.get_attribute( node, cls_val, "__new__") new_attr = abstract_utils.get_atomic_value(new_attr) node = self.vm.attribute_handler.set_attribute( node, new_attr, "__defaults__", defaults) # set the attribute without overriding special namedtuple attributes node, fields = self.vm.attribute_handler.get_attribute( node, cls_val, "_fields") fields = abstract_utils.get_atomic_python_constant(fields, tuple) fields = [ abstract_utils.get_atomic_python_constant(field, str) for field in fields ] for key in f_locals: if key in self._prohibited: self.vm.errorlog.not_writable(self.vm.frames, cls_val, key) if key not in self._special and key not in fields: node = self.vm.attribute_handler.set_attribute( node, cls_val, key, f_locals[key]) return node, cls_var
def test_getitem__concrete_index(self): t = abstract.Tuple((self._var, ), self._vm) index = self._vm.convert.constant_to_var(0) node, var = t.cls.getitem_slot(self._node, index) self.assertIs(node, self._node) self.assertIs(abstract_utils.get_atomic_value(var), abstract_utils.get_atomic_value(self._var))
def make_class(self, node, f_locals): f_locals = abstract_utils.get_atomic_python_constant(f_locals) # retrieve __qualname__ to get the name of class name = f_locals["__qualname__"] # retrieve __annotations__ to get the dict # with key-value pair of (variable, type) anno = f_locals.get("__annotations__", {}) if anno: anno = abstract_utils.get_atomic_value(anno) # assemble the arguments that are compatible with NamedTupleFuncBuilder.call field_list = [] defaults = [] for k, v in anno.items(): if k in f_locals: defaults.append(f_locals.get(k)) # TODO(ahxun): check if the value matches the declared type k = self.vm.convert.constant_to_var(k, node=node) field_list.append(self.vm.convert.build_tuple(node, (k, v))) anno = self.vm.convert.build_list(node, field_list) posargs = (name, anno) args = function.Args(posargs=posargs) node, cls_var = self.namedtuple.call(node, None, args) cls_val = abstract_utils.get_atomic_value(cls_var) if not isinstance(cls_val, abstract.Unsolvable): # set __new__.__defaults__ defaults = abstract.Tuple(tuple(defaults), self.vm).to_variable(node) node, new_attr = self.vm.attribute_handler.get_attribute( node, cls_val, "__new__") new_attr = abstract_utils.get_atomic_value(new_attr) node = self.vm.attribute_handler.set_attribute( node, new_attr, "__defaults__", defaults) # set the attribute without overriding special namedtuple attributes node, fields = self.vm.attribute_handler.get_attribute( node, cls_val, "_fields") fields = abstract_utils.get_atomic_python_constant(fields, tuple) fields = [ abstract_utils.get_atomic_python_constant(field, str) for field in fields ] for key in f_locals: if key in self._prohibited: self.vm.errorlog.not_writable(self.vm.frames, cls_val, key) if key not in self._special and key not in fields: node = self.vm.attribute_handler.set_attribute( node, cls_val, key, f_locals[key]) return node, cls_var
def _eval_expr_as_tuple(self, node, f_globals, f_locals, expr): """Evaluate an expression as a tuple.""" if not expr: return () result = abstract_utils.get_atomic_value( self._eval_expr(node, f_globals, f_locals, expr)) # If the result is a tuple, expand it. if (isinstance(result, mixin.PythonConstant) and isinstance(result.pyval, tuple)): return tuple(abstract_utils.get_atomic_value(x) for x in result.pyval) else: return (result,)
def test_instantiate_interpreter_class(self): cls = abstract.InterpreterClass("X", [], {}, None, self._vm) # When there is no current frame, create a new instance every time. v1 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) v2 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) self.assertIsNot(v1, v2) # Create one instance per opcode. fake_opcode = object() self._vm.push_frame(frame_state.SimpleFrame(fake_opcode)) v3 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) v4 = abstract_utils.get_atomic_value(cls.instantiate(self._node)) self.assertIsNot(v1, v3) self.assertIsNot(v2, v3) self.assertIs(v3, v4)
def call(self, node, unused_f, args, alias_map=None): _, argmap = self.match_and_map_args(node, args, alias_map) this_var = argmap["self"] other_var = argmap["other"] # This is called by vm._call_binop_on_bindings, so both should have # exactly 1 possibility. try: this = abstract_utils.get_atomic_value(this_var) other = abstract_utils.get_atomic_value(other_var) except abstract_utils.ConversionError: return node, self.vm.convert.build_bool(node) return node, self.vm.convert.build_bool( node, this.cls == other.cls and "name" in this.members and this.members["name"] == other.members.get("name"))
def _eval_expr_as_tuple(self, node, expr, stack): """Evaluate an expression as a tuple.""" if not expr: return () f_globals, f_locals = self.vm.frame.f_globals, self.vm.frame.f_locals with self.vm.generate_late_annotations(stack): result = abstract_utils.get_atomic_value( abstract_utils.eval_expr(self.vm, node, f_globals, f_locals, expr)) # If the result is a tuple, expand it. if (isinstance(result, mixin.PythonConstant) and isinstance(result.pyval, tuple)): return tuple(abstract_utils.get_atomic_value(x) for x in result.pyval) else: return (result,)
def apply_type_comment(self, state, op, name, value): """If there is a type comment for the op, return its value.""" assert op is self.vm.frame.current_opcode if op.code.co_filename != self.vm.filename: return value if not op.type_comment: return value comment = op.type_comment try: var = self._eval_expr(state.node, self.vm.frame.f_globals, self.vm.frame.f_locals, comment) except EvaluationError as e: self.vm.errorlog.invalid_type_comment(self.vm.frames, comment, details=utils.message(e)) value = self.vm.new_unsolvable(state.node) else: try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.invalid_type_comment( self.vm.frames, comment, details="Must be constant.") value = self.vm.new_unsolvable(state.node) else: if self.get_type_parameters(typ): self.vm.errorlog.not_supported_yet( self.vm.frames, "using type parameter in type comment") try: value = self.init_annotation(typ, name, self.vm.frames, state.node) except self.LateAnnotationError: value = LateAnnotation(typ, name, self.vm.simple_stack()) return value
def call(self, node, unused_func, args): """Adds a metaclass.""" self.match_args(node, args) meta = abstract_utils.get_atomic_value( args.posargs[0], default=self.vm.convert.unsolvable) return node, AddMetaclassInstance( meta, self.vm, self.module_name).to_variable(node)
def call(self, node, func, args): args = args.simplify(node) self.match_args(node, args, match_all_views=True) # As long as the types match we do not really care about the actual # class name. But, if we have a string literal value as the name arg, # we will use it. name_arg = args.namedargs.get(self._name_arg_name) or args.posargs[0] try: _ = abstract_utils.get_atomic_python_constant(name_arg, str) except abstract_utils.ConversionError: name_arg = self.vm.convert.constant_to_var( "_NewType_Internal_Class_Name_%d_" % self.internal_name_counter) type_arg = args.namedargs.get(self._type_arg_name) or args.posargs[1] try: type_value = abstract_utils.get_atomic_value(type_arg) except abstract_utils.ConversionError: # We need the type arg to be an atomic value. If not, we just # silently return unsolvable. return node, self.vm.new_unsolvable(node) value_arg_name = "val" constructor = overlay_utils.make_method( self.vm, node, name="__init__", params=[Param(value_arg_name, type_value)]) members = abstract.Dict(self.vm) members.set_str_item(node, "__init__", constructor) return self.vm.make_class(node, name_arg, (type_arg, ), members.to_variable(node), None)
def apply_type_comment(self, state, op, name, value): """If there is a type comment for the op, return its value.""" assert op is self.vm.frame.current_opcode if op.code.co_filename != self.vm.filename: return value if not op.type_comment: return value comment = op.type_comment frame = self.vm.frame var, errorlog = abstract_utils.eval_expr(self.vm, state.node, frame.f_globals, frame.f_locals, comment) if errorlog: self.vm.errorlog.invalid_type_comment(self.vm.frames, comment, details=errorlog.details) try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.invalid_type_comment(self.vm.frames, comment, details="Must be constant.") value = self.vm.new_unsolvable(state.node) else: typ = self._process_one_annotation(state.node, typ, name, self.vm.simple_stack()) if typ: if self.get_type_parameters(typ): self.vm.errorlog.not_supported_yet( self.vm.frames, "using type parameter in type comment") _, value = self.vm.init_class(state.node, typ) else: value = self.vm.new_unsolvable(state.node) return value
def _getargs(self, node, args): self.match_args(node, args) sig, = self.signatures callargs = { name: var for name, var, _ in sig.signature.iter_args(args) } # typing.NamedTuple doesn't support rename or verbose name_var = callargs["typename"] fields_var = callargs["fields"] fields = abstract_utils.get_atomic_python_constant(fields_var) # The fields is a list of tuples, so we need to deeply unwrap them. fields = [abstract_utils.get_atomic_python_constant(t) for t in fields] # We need the actual string for the field names and the AtomicAbstractValue # for the field types. names = [] types = [] for field in fields: if (len(field) != 2 or any(not self._is_str_instance(v) for v in field[0].data)): # Note that we don't need to check field[1] because both 'str' # (forward reference) and 'type' are valid for it. sig, = self.signatures bad_param = function.BadParam(name="fields", expected=self._fields_type) raise function.WrongArgTypes(sig.signature, args, self.vm, bad_param) name, typ = field names.append(abstract_utils.get_atomic_python_constant(name)) types.append(abstract_utils.get_atomic_value(typ)) return name_var, names, types
def call(self, node, unused_func, args): """Creates an anonymous class to act as a metaclass.""" self.match_args(node, args) meta = abstract_utils.get_atomic_value( args.posargs[0], default=self.vm.convert.unsolvable) bases = args.posargs[1:] result = WithMetaclassInstance(self.vm, meta, bases).to_variable(node) return node, result
def test_call_wrong_argcount(self): self._vm.push_frame(frame_state.SimpleFrame()) node, result = self._is_instance.call( self._node, None, function.Args((), self.new_dict(), None, None)) self.assertEqual(self._node, node) self.assertIsInstance(abstract_utils.get_atomic_value(result), abstract.Unsolvable) six.assertRegex(self, str(self._vm.errorlog), "missing-parameter")
def extract_annotation( self, node, var, name, stack, allowed_type_params=None, use_not_supported_yet=True): """Returns an annotation extracted from 'var'. Args: node: The current node. var: The variable to extract from. name: The annotated name. stack: The frame stack. allowed_type_params: Type parameters that are allowed to appear in the annotation. 'None' means all are allowed. use_not_supported_yet: Temporary parameter to help transition the error class for reporting 'type parameter not in scope' errors from [not-supported-yet] to [invalid-annotation]. """ try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.ambiguous_annotation(self.vm.frames, None, name) return self.vm.convert.unsolvable typ = self._process_one_annotation(node, typ, name, stack) if not typ: return self.vm.convert.unsolvable if typ.formal and allowed_type_params is not None: illegal_params = [x.name for x in self.get_type_parameters(typ) if x.name not in allowed_type_params] if illegal_params: details = "TypeVar(s) %s not in scope" % ", ".join( repr(p) for p in utils.unique_list(illegal_params)) if self.vm.frame.func: method = self.vm.frame.func.data if isinstance(method, abstract.BoundFunction): desc = "class" frame_name = method.name.rsplit(".", 1)[0] else: desc = "class" if method.is_class_builder else "method" frame_name = method.name details += f" for {desc} {frame_name!r}" if "AnyStr" in illegal_params: if self.vm.PY2: str_type = "typing.Text" else: str_type = "Union[str, bytes]" details += ( f"\nNote: For all string types, use {str_type}.") if use_not_supported_yet: # TODO(b/186896951): Switch this to an [invalid-annotation] error. self.vm.errorlog.not_supported_yet( stack, "using type parameter in variable annotation", details=details) else: self.vm.errorlog.invalid_annotation(stack, typ, details, name) return self.vm.convert.unsolvable return typ
def test_call_wrong_keywords(self): self._vm.push_frame(frame_state.SimpleFrame()) x = self.new_var(abstract.Unknown(self._vm)) node, result = self._is_instance.call( self._node, None, function.Args( (x, x), self.new_dict(foo=x), None, None)) self.assertEqual(self._node, node) self.assertIsInstance(abstract_utils.get_atomic_value(result), abstract.Unsolvable) six.assertRegex(self, str(self._vm.errorlog), r"foo.*isinstance.*\[wrong-keyword-args\]")
def _get_annotation(self, var, name): try: ret = abstract_utils.get_atomic_value(var, self._CLASS_TYPE) if isinstance(ret, abstract.AbstractOrConcreteValue): ret = abstract_utils.get_atomic_python_constant( var, six.string_types) except abstract_utils.ConversionError: raise TypeVarError("%s must be constant" % name) if not ret: raise TypeVarError("%s cannot be an empty string" % name) return ret
def _get_annotation(self, node, var, name): with self.vm.errorlog.checkpoint() as record: retvar = self.vm.annotations_util.process_annotation_var( node, var, name, self.vm.simple_stack()) if record.errors: raise TypeVarError("\n".join(error.message for error in record.errors)) try: return abstract_utils.get_atomic_value(retvar) except abstract_utils.ConversionError: raise TypeVarError("%s must be constant" % name)
def _load_all_formal_type_parameters(self): """Load _all_formal_type_parameters.""" if self._all_formal_type_parameters_loaded: return bases = [ abstract_utils.get_atomic_value( base, default=self.vm.convert.unsolvable) for base in self.bases()] for base in bases: abstract_utils.parse_formal_type_parameters( base, self.full_name, self._all_formal_type_parameters) self._all_formal_type_parameters_loaded = True
def extract_annotation(self, node, var, name, stack, is_var=False): try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.ambiguous_annotation(self.vm.frames, None, name) return self.vm.convert.unsolvable typ = self._process_one_annotation(node, typ, name, stack) if not typ: return self.vm.convert.unsolvable if typ.formal and is_var: self.vm.errorlog.not_supported_yet( stack, "using type parameter in variable annotation") return self.vm.convert.unsolvable return typ
def test_callable_no_args(self): ast = self._load_ast("a", """ from typing import Callable x = ... # type: Callable[[], ...] """) x = ast.Lookup("a.x").type cls = self._vm.convert.constant_to_value(x, {}, self._vm.root_cfg_node) instance = self._vm.convert.constant_to_value( abstract_utils.AsInstance(x), {}, self._vm.root_cfg_node) self.assertIsInstance( cls.get_formal_type_parameter(abstract_utils.ARGS), abstract.Empty) self.assertEqual(abstract_utils.get_atomic_value( instance.get_instance_type_parameter(abstract_utils.ARGS)), self._vm.convert.empty)
def init_annotation_var(self, node, name, var): """Instantiate a variable of an annotation, calling __init__.""" try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: error = "Type must be constant for variable annotation" self.vm.errorlog.invalid_annotation(self.vm.frames, None, error, name) return self.vm.new_unsolvable(node) else: if self.get_type_parameters(typ): self.vm.errorlog.not_supported_yet( self.vm.frames, "using type parameter in variable annotation") return self.vm.new_unsolvable(node) _, instance = self.vm.init_class(node, typ) return instance
def extract_annotation( self, node, var, name, stack, allowed_type_params=None): """Returns an annotation extracted from 'var'. Args: node: The current node. var: The variable to extract from. name: The annotated name. stack: The frame stack. allowed_type_params: Type parameters that are allowed to appear in the annotation. 'None' means all are allowed. """ try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.ambiguous_annotation(self.vm.frames, None, name) return self.vm.convert.unsolvable typ = self._process_one_annotation(node, typ, name, stack) if not typ: return self.vm.convert.unsolvable if typ.formal and allowed_type_params is not None: illegal_params = [x.name for x in self.get_type_parameters(typ) if x.name not in allowed_type_params] if illegal_params: details = "TypeVar(s) %s not in scope" % ", ".join( repr(p) for p in utils.unique_list(illegal_params)) if self.vm.frame.func: method = self.vm.frame.func.data if isinstance(method, abstract.BoundFunction): class_name = method.name.rsplit(".", 1)[0] else: class_name = method.name details += " for class %r" % class_name if "AnyStr" in illegal_params: if self.vm.PY2: str_type = "typing.Text" else: str_type = "Union[str, bytes]" details += ( f"\nNote: For all string types, use {str_type}.") # TODO(b/186896951): Once the remaining use cases in the linked bug are # supported, we should switch this to an [invalid-annotation] error. self.vm.errorlog.not_supported_yet( stack, "using type parameter in variable annotation", details=details) return self.vm.convert.unsolvable return typ
def type_to_value(self, node, name, type_var): """Convert annotation type to instance value.""" try: typ = abstract_utils.get_atomic_value(type_var) except abstract_utils.ConversionError: error = "Type must be constant for variable annotation" self.vm.errorlog.invalid_annotation(self.vm.frames, None, error, name) return self.vm.convert.create_new_unsolvable(node) else: if self.get_type_parameters(typ): self.vm.errorlog.not_supported_yet( self.vm.frames, "using type parameter in variable annotation") return self.vm.convert.create_new_unsolvable(node) try: return self.init_annotation(typ, name, self.vm.frames, node) except self.LateAnnotationError: return LateAnnotation(typ, name, self.vm.simple_stack())
def _getargs(self, node, args): self.match_args(node, args) sig, = self.signatures callargs = { name: var for name, var, _ in sig.signature.iter_args(args) } # typing.NamedTuple doesn't support rename or verbose name_var = callargs["typename"] fields_var = callargs["fields"] fields = abstract_utils.get_atomic_python_constant(fields_var) if isinstance(fields, six.string_types): # Since str matches Sequence, we have to manually check for it. raise function.WrongArgTypes(sig.signature, args, self.vm, self._fields_param) # The fields is a list of tuples, so we need to deeply unwrap them. fields = [abstract_utils.get_atomic_python_constant(t) for t in fields] # We need the actual string for the field names and the AtomicAbstractValue # for the field types. names = [] types = [] for field in fields: if isinstance(field, six.string_types): # Since str matches Sequence, we have to manually check for it. raise function.WrongArgTypes(sig.signature, args, self.vm, self._fields_param) if (len(field) != 2 or any(not self._is_str_instance(v) for v in field[0].data)): # Note that we don't need to check field[1] because both 'str' # (forward reference) and 'type' are valid for it. raise function.WrongArgTypes(sig.signature, args, self.vm, self._fields_param) name, typ = field name_py_constant = abstract_utils.get_atomic_python_constant(name) if name_py_constant.__class__ is compat.UnicodeType: # Unicode values should be ASCII. name_py_constant = compat.native_str( name_py_constant.encode("ascii")) names.append(name_py_constant) types.append(abstract_utils.get_atomic_value(typ)) return name_var, names, types
def call(self, node, _, args, alias_map=None): _, argmap = self.match_and_map_args(node, args, alias_map) cls_var = argmap["cls"] name_var = argmap["name"] try: cls = abstract_utils.get_atomic_value(cls_var) except abstract_utils.ConversionError: return node, self.vm.new_unsolvable(node) # If we can't get a concrete name, treat it like it matches and return a # canonical enum member. try: name = abstract_utils.get_atomic_python_constant(name_var, str) except abstract_utils.ConversionError: return node, cls.instantiate(node) inst = self._get_member_by_name(cls, name) if inst: return node, inst else: self.vm.errorlog.attribute_error(self.vm.frames, cls_var.bindings[0], name) return node, self.vm.new_unsolvable(node)
def extract_annotation(self, node, var, name, stack, allowed_type_params=None): """Returns an annotation extracted from 'var'. Args: node: The current node. var: The variable to extract from. name: The annotated name. stack: The frame stack. allowed_type_params: Type parameters that are allowed to appear in the annotation. 'None' means all are allowed. """ try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.ambiguous_annotation(self.vm.frames, None, name) return self.vm.convert.unsolvable typ = self._process_one_annotation(node, typ, name, stack) if not typ: return self.vm.convert.unsolvable if typ.formal and allowed_type_params is not None: if "AnyStr" in [x.name for x in self.get_type_parameters(typ)]: if self.vm.PY2: str_type = "typing.Text" else: str_type = "Union[str, bytes]" details = f"Note: AnyStr is a TypeVar; use {str_type} for string types." else: details = None self.vm.errorlog.not_supported_yet( stack, "using type parameter in variable annotation", details=details) return self.vm.convert.unsolvable return typ
def extract_annotation(self, node, var, name, stack, is_var=False): try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.ambiguous_annotation(self.vm.frames, None, name) return self.vm.convert.unsolvable typ = self._process_one_annotation(node, typ, name, stack) if not typ: return self.vm.convert.unsolvable if typ.formal and is_var: if "AnyStr" in [x.name for x in self.get_type_parameters(typ)]: if self.vm.PY2: str_type = "typing.Text" else: str_type = "Union[str, bytes]" details = f"Note: AnyStr is a TypeVar; use {str_type} for string types." else: details = None self.vm.errorlog.not_supported_yet( stack, "using type parameter in variable annotation", details=details) return self.vm.convert.unsolvable return typ
def _static_method_to_def(self, node, v, name, kind): """Convert a staticmethod to a PyTD definition.""" # This line may raise abstract_utils.ConversionError func = abstract_utils.get_atomic_value(v.func, abstract.Function) return self.value_to_pytd_def(node, func, name).Replace(kind=kind)