def _getargs(self, node, args):
"""Extracts the typename, field_names and rename arguments.
collections.namedtuple takes potentially 4 arguments, but we only care about
three of them. This function checks the argument count and ensures multiple
values aren't passed for 'verbose' and 'rename'.
Args:
node: The current CFG node. Used by match_args.
args: A function.Args object
Returns:
A tuple containing the typename, field_names and rename arguments passed
to this call to collections.namedtuple.
Raises:
function.FailedFunctionCall: The arguments do not match those needed by
the function call. See also: abstract.PyTDFunction.match_args().
abstract_utils.ConversionError: One of the args could not be extracted.
Typically occurs if typename or one of the field names is in unicode.
"""
# abstract.PyTDFunction.match_args checks the args for this call.
self.match_args(node, args)
# namedtuple only has one signature
sig, = self.signatures
callargs = {name: var for name, var, _ in sig.signature.iter_args(args)}
# The name of the namedtuple class is the first arg (a Variable)
# We need the actual Variable later, so we'll just return name_var and
# extract the name itself later.
name_var = callargs["typename"]
# The fields are also a Variable, which stores the field names as Variables.
# Extract the list itself, we don't need the wrapper.
fields_var = callargs["field_names"]
fields = abstract_utils.get_atomic_python_constant(fields_var)
# namedtuple fields can be given as a single string, e.g. "a, b, c" or as a
# list [Variable('a'), Variable('b'), Variable('c')].
# We just want a list of strings.
if isinstance(fields, (bytes, six.text_type)):
fields = compat.native_str(fields)
field_names = fields.replace(",", " ").split()
else:
field_names = [abstract_utils.get_atomic_python_constant(f)
for f in fields]
field_names = [compat.native_str(f) for f in field_names]
# namedtuple also takes a "verbose" argument, but we don't care about that.
# rename will take any problematic field names and give them a new name.
# Like the other args, it's stored as a Variable, but we want just a bool.
if callargs.get("rename", None):
rename = abstract_utils.get_atomic_python_constant(callargs["rename"])
else:
rename = False
return name_var, field_names, rename
def _match_pyval_against_string(self, pyval, string, subst):
"""Matches a concrete value against a string literal."""
assert isinstance(string, str)
if pyval.__class__ is str: # native str
left_type = "bytes" if self.vm.PY2 else "unicode"
elif isinstance(pyval, compat.BytesType):
left_type = "bytes"
elif isinstance(pyval, compat.UnicodeType):
left_type = "unicode"
else:
return None
# needs to be native str to match `string`
left_value = compat.native_str(pyval)
right_prefix, right_value = (
parser_constants.STRING_RE.match(string).groups()[:2])
if "b" in right_prefix or "u" not in right_prefix and self.vm.PY2:
right_type = "bytes"
else:
right_type = "unicode"
right_value = right_value[1:-1] # remove quotation marks
if left_type == right_type and left_value == right_value:
return subst
return None
def compile_src_string_to_pyc_string(src, filename, python_exe, mode="exec"):
"""Compile Python source code to pyc data.
This may use py_compile if the src is for the same version as we're running,
or else it spawns an external process to produce a .pyc file. The generated
bytecode (.pyc file) is read and both it and any temporary files are deleted.
Args:
src: Python sourcecode
filename: Name of the source file. For error messages.
python_exe: Tuple of a path to a Python interpreter and command-line flags.
mode: Same as __builtin__.compile: "exec" if source consists of a
sequence of statements, "eval" if it consists of a single expression,
or "single" if it consists of a single interactive statement.
Returns:
The compiled pyc file as a binary string.
Raises:
CompileError: If we find a syntax error in the file.
IOError: If our compile script failed.
"""
tempfile_options = {"mode": "w", "suffix": ".py", "delete": False}
if six.PY3:
tempfile_options.update({"encoding": "utf-8"})
else:
tempfile_options.update({"mode": "wb"})
fi = tempfile.NamedTemporaryFile(**tempfile_options)
try:
if six.PY3:
fi.write(src)
else:
fi.write(src.encode("utf-8"))
fi.close()
# In order to be able to compile pyc files for both Python 2 and Python 3,
# we spawn an external process.
# We pass -E to ignore the environment so that PYTHONPATH and sitecustomize
# on some people's systems don't mess with the interpreter.
exe, flags = python_exe
cmd = [exe] + flags + ["-E", "-", fi.name, filename or fi.name, mode]
compile_script_src = pytype_source_utils.load_pytype_file(
COMPILE_SCRIPT)
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
bytecode, _ = p.communicate(compile_script_src)
assert p.poll() == 0, "Child process failed"
finally:
os.unlink(fi.name)
first_byte = six.indexbytes(bytecode, 0)
if first_byte == 0: # compile OK
return bytecode[1:]
elif first_byte == 1: # compile error
code = bytecode[1:] # type: bytes
raise CompileError(compat.native_str(code))
else:
raise IOError("_compile.py produced invalid result")
def load_string(self):
n = self._read_long()
s = self._read(n)
if self.python_version[0] >= 3:
# load_string() loads a bytestring, and in Python 3, str and bytes are
# different classes.
s = compat.BytesType(s)
elif not self._keep_bytes:
# In Python 2, load_string is used to load both bytestrings and native
# strings, so we have to specify which we want.
s = compat.native_str(s)
return s
def load_string(self):
n = self._read_long()
s = self._read(n)
if self.python_version[0] >= 3:
# load_string() loads a bytestring, and in Python 3, str and bytes are
# different classes.
s = compat.BytesType(s)
elif not self._keep_bytes:
# In Python 2, load_string is used to load both bytestrings and native
# strings, so we have to specify which we want. We use the
# 'backslashreplace' error mode in order to handle non-utf8
# backslash-escaped string literals correctly.
s = compat.native_str(s, 'backslashreplace')
return s
def _getargs(self, node, args):
self.match_args(node, args)
sig, = self.signatures
callargs = {
name: var
for name, var, _ in sig.signature.iter_args(args)
}
# typing.NamedTuple doesn't support rename or verbose
name_var = callargs["typename"]
fields_var = callargs["fields"]
fields = abstract_utils.get_atomic_python_constant(fields_var)
if isinstance(fields, six.string_types):
# Since str matches Sequence, we have to manually check for it.
raise function.WrongArgTypes(sig.signature, args, self.vm,
self._fields_param)
# The fields is a list of tuples, so we need to deeply unwrap them.
fields = [abstract_utils.get_atomic_python_constant(t) for t in fields]
# We need the actual string for the field names and the AtomicAbstractValue
# for the field types.
names = []
types = []
for field in fields:
if isinstance(field, six.string_types):
# Since str matches Sequence, we have to manually check for it.
raise function.WrongArgTypes(sig.signature, args, self.vm,
self._fields_param)
if (len(field) != 2 or any(not self._is_str_instance(v)
for v in field[0].data)):
# Note that we don't need to check field[1] because both 'str'
# (forward reference) and 'type' are valid for it.
raise function.WrongArgTypes(sig.signature, args, self.vm,
self._fields_param)
name, typ = field
name_py_constant = abstract_utils.get_atomic_python_constant(name)
if name_py_constant.__class__ is compat.UnicodeType:
# Unicode values should be ASCII.
name_py_constant = compat.native_str(
name_py_constant.encode("ascii"))
names.append(name_py_constant)
types.append(abstract_utils.get_atomic_value(typ))
return name_var, names, types
def _build_namedtuple(self, name, field_names, field_types, late_annots,
node):
# Build an InterpreterClass representing the namedtuple.
if field_types:
# TODO(mdemello): Fix this to support late types.
field_types_union = abstract.Union(field_types, self.vm)
else:
field_types_union = self.vm.convert.none_type
members = {
n: t.instantiate(node)
for n, t in moves.zip(field_names, field_types)
}
# collections.namedtuple has: __dict__, __slots__ and _fields.
# typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults.
# __slots__ and _fields are tuples containing the names of the fields.
slots = tuple(
self.vm.convert.build_string(node, f) for f in field_names)
members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node)
members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node)
# __dict__ and _field_defaults are both collections.OrderedDicts that map
# field names (strings) to objects of the field types.
ordered_dict_cls = self.vm.convert.name_to_value(
"collections.OrderedDict", ast=self.collections_ast)
# In Python 2, keys can be `str` or `unicode`; support both.
# In Python 3, `str_type` and `unicode_type` are the same.
field_keys_union = abstract.Union(
[self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm)
# Normally, we would use abstract_utils.K and abstract_utils.V, but
# collections.pyi doesn't conform to that standard.
field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, {
"K": field_keys_union,
"V": field_types_union
}, self.vm)
members["__dict__"] = field_dict_cls.instantiate(node)
members["_field_defaults"] = field_dict_cls.instantiate(node)
# _field_types and __annotations__ are both collections.OrderedDicts
# that map field names (strings) to the types of the fields.
field_types_cls = abstract.ParameterizedClass(
ordered_dict_cls, {
"K": field_keys_union,
"V": self.vm.convert.type_type
}, self.vm)
members["_field_types"] = field_types_cls.instantiate(node)
members["__annotations__"] = field_types_cls.instantiate(node)
# __new__
# We set the bound on this TypeParameter later. This gives __new__ the
# signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same
# signature that visitor.CreateTypeParametersForSignatures would create.
# This allows subclasses of the NamedTuple to get the correct type from
# their constructors.
cls_type_param = abstract.TypeParameter(
visitors.CreateTypeParametersForSignatures.PREFIX + name,
self.vm,
bound=None)
cls_type = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
# Use late annotations as field types if they exist.
params = [
Param(n, late_annots.get(n, t))
for n, t in moves.zip(field_names, field_types)
]
members["__new__"] = overlay_utils.make_method(
self.vm,
node,
name="__new__",
self_param=Param("cls", cls_type),
params=params,
return_type=cls_type_param,
)
# __init__
members["__init__"] = overlay_utils.make_method(self.vm,
node,
name="__init__",
varargs=Param("args"),
kwargs=Param("kwargs"))
# _make
# _make is a classmethod, so it needs to be wrapped by
# specialibuiltins.ClassMethodInstance.
# Like __new__, it uses the _Tname TypeVar.
sized_cls = self.vm.convert.name_to_value("typing.Sized")
iterable_type = abstract.ParameterizedClass(
self.vm.convert.name_to_value("typing.Iterable"),
{abstract_utils.T: field_types_union}, self.vm)
cls_type = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
len_type = abstract.CallableClass(
self.vm.convert.name_to_value("typing.Callable"), {
0: sized_cls,
abstract_utils.ARGS: sized_cls,
abstract_utils.RET: self.vm.convert.int_type
}, self.vm)
params = [
Param("iterable", iterable_type),
Param("new").unsolvable(self.vm, node),
Param("len", len_type).unsolvable(self.vm, node)
]
make = overlay_utils.make_method(self.vm,
node,
name="_make",
params=params,
self_param=Param("cls", cls_type),
return_type=cls_type_param)
make_args = function.Args(posargs=(make, ))
_, members["_make"] = self.vm.special_builtins["classmethod"].call(
node, None, make_args)
# _replace
# Like __new__, it uses the _Tname TypeVar. We have to annotate the `self`
# param to make sure the TypeVar is substituted correctly.
members["_replace"] = overlay_utils.make_method(
self.vm,
node,
name="_replace",
self_param=Param("self", cls_type_param),
return_type=cls_type_param,
kwargs=Param("kwds", field_types_union))
# __getnewargs__
getnewargs_tuple_params = dict(
tuple(enumerate(field_types)) +
((abstract_utils.T, field_types_union), ))
getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type,
getnewargs_tuple_params,
self.vm)
members["__getnewargs__"] = overlay_utils.make_method(
self.vm, node, name="__getnewargs__", return_type=getnewargs_tuple)
# __getstate__
members["__getstate__"] = overlay_utils.make_method(
self.vm, node, name="__getstate__")
# _asdict
members["_asdict"] = overlay_utils.make_method(
self.vm, node, name="_asdict", return_type=field_dict_cls)
# Finally, make the class.
cls_dict = abstract.Dict(self.vm)
cls_dict.update(node, members)
if name.__class__ is compat.UnicodeType:
# Unicode values should be ASCII.
name = compat.native_str(name.encode("ascii"))
node, cls_var = self.vm.make_class(
node=node,
name_var=self.vm.convert.build_string(node, name),
bases=[self.vm.convert.tuple_type.to_variable(node)],
class_dict_var=cls_dict.to_variable(node),
cls_var=None)
cls = cls_var.data[0]
# Now that the class has been made, we can complete the TypeParameter used
# by __new__, _make and _replace.
cls_type_param.bound = cls
# Add late annotations to the new class
if late_annots:
cls.late_annotations = late_annots
self.vm.classes_with_late_annotations.append(cls)
return node, cls_var
def compile_src_string_to_pyc_string(src,
filename,
python_version,
python_exe,
mode="exec"):
"""Compile Python source code to pyc data.
This may use py_compile if the src is for the same version as we're running,
or else it spawns an external process to produce a .pyc file. The generated
bytecode (.pyc file) is read and both it and any temporary files are deleted.
Args:
src: Python sourcecode
filename: Name of the source file. For error messages.
python_version: Python version, (major, minor).
python_exe: Tuple of a path to a Python interpreter and command-line flags.
mode: Same as __builtin__.compile: "exec" if source consists of a
sequence of statements, "eval" if it consists of a single expression,
or "single" if it consists of a single interactive statement.
Returns:
The compiled pyc file as a binary string.
Raises:
CompileError: If we find a syntax error in the file.
IOError: If our compile script failed.
"""
if python_version == sys.version_info[:2] and (
sys.version_info.major != 2 or not utils.USE_ANNOTATIONS_BACKPORT):
# Optimization: calling compile_bytecode directly is faster than spawning a
# subprocess. We can do this only when the host and target versions match
# and we don't need the patched 2.7 interpreter.
output = six.BytesIO()
compile_bytecode.compile_src_to_pyc(src, filename or "<>", output,
mode)
bytecode = output.getvalue()
else:
tempfile_options = {"mode": "w", "suffix": ".py", "delete": False}
if six.PY3:
tempfile_options.update({"encoding": "utf-8"})
else:
tempfile_options.update({"mode": "wb"})
fi = tempfile.NamedTemporaryFile(**tempfile_options)
try:
if six.PY3:
fi.write(src)
else:
fi.write(src.encode("utf-8"))
fi.close()
# In order to be able to compile pyc files for a different Python version
# from the one we're running under, we spawn an external process.
# We pass -E to ignore the environment so that PYTHONPATH and
# sitecustomize on some people's systems don't mess with the interpreter.
exe, flags = python_exe
cmd = [exe
] + flags + ["-E", "-", fi.name, filename or fi.name, mode]
compile_script_src = pytype_source_utils.load_pytype_file(
COMPILE_SCRIPT)
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
bytecode, _ = p.communicate(compile_script_src)
assert p.poll() == 0, "Child process failed"
finally:
os.unlink(fi.name)
first_byte = six.indexbytes(bytecode, 0)
if first_byte == 0: # compile OK
return bytecode[1:]
elif first_byte == 1: # compile error
code = bytecode[1:] # type: bytes
raise CompileError(compat.native_str(code))
else:
raise IOError("_compile.py produced invalid result")
def load_short_ascii(self):
n = self._read_byte()
return compat.native_str(self._read(n))
def load_ascii(self):
n = self._read_long()
return compat.native_str(self._read(n))
def load_interned(self):
n = self._read_long()
s = self._read(n)
ret = six.moves.intern(compat.native_str(s))
self._stringtable.append(ret)
return ret
def _build_namedtuple(self, name, field_names, field_types, node):
# Build an InterpreterClass representing the namedtuple.
if field_types:
field_types_union = abstract.Union(field_types, self.vm)
else:
field_types_union = self.vm.convert.none_type
members = {
n: t.instantiate(node)
for n, t in moves.zip(field_names, field_types)
}
# collections.namedtuple has: __dict__, __slots__ and _fields.
# typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults.
# __slots__ and _fields are tuples containing the names of the fields.
slots = tuple(
self.vm.convert.build_string(node, f) for f in field_names)
members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node)
members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node)
# __dict__ and _field_defaults are both collections.OrderedDicts that map
# field names (strings) to objects of the field types.
ordered_dict_cls = self.vm.convert.name_to_value(
"collections.OrderedDict", ast=self.collections_ast)
# In Python 2, keys can be `str` or `unicode`; support both.
# In Python 3, `str_type` and `unicode_type` are the same.
field_keys_union = abstract.Union(
[self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm)
# Normally, we would use abstract_utils.K and abstract_utils.V, but
# collections.pyi doesn't conform to that standard.
field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, {
"K": field_keys_union,
"V": field_types_union
}, self.vm)
members["__dict__"] = field_dict_cls.instantiate(node)
members["_field_defaults"] = field_dict_cls.instantiate(node)
# _field_types and __annotations__ are both collections.OrderedDicts
# that map field names (strings) to the types of the fields.
field_types_cls = abstract.ParameterizedClass(
ordered_dict_cls, {
"K": field_keys_union,
"V": self.vm.convert.type_type
}, self.vm)
members["_field_types"] = field_types_cls.instantiate(node)
members["__annotations__"] = field_types_cls.instantiate(node)
# __new__
new_annots = {}
new_lates = {}
for (n, t) in moves.zip(field_names, field_types):
# We don't support late annotations yet, but once we do, they'll show up
# as LateAnnotation objects to be stored in new_lates.
new_annots[n] = t
# We set the bound on this TypeParameter later. This gives __new__ the
# signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same
# signature that visitor.CreateTypeParametersForSignatures would create.
# This allows subclasses of the NamedTuple to get the correct type from
# their constructors.
cls_type_param = abstract.TypeParameter(
visitors.CreateTypeParametersForSignatures.PREFIX + name,
self.vm,
bound=None)
new_annots["cls"] = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
new_annots["return"] = cls_type_param
members["__new__"] = abstract.SimpleFunction(
name="__new__",
param_names=("cls", ) + tuple(field_names),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations=new_annots,
late_annotations=new_lates,
vm=self.vm).to_variable(node)
# __init__
members["__init__"] = abstract.SimpleFunction(
name="__init__",
param_names=("self", ),
varargs_name="args",
kwonly_params=(),
kwargs_name="kwargs",
defaults={},
annotations={},
late_annotations={},
vm=self.vm).to_variable(node)
# _make
# _make is a classmethod, so it needs to be wrapped by
# specialibuiltins.ClassMethodInstance.
# Like __new__, it uses the _Tname TypeVar.
sized_cls = self.vm.convert.name_to_value("typing.Sized")
iterable_type = abstract.ParameterizedClass(
self.vm.convert.name_to_value("typing.Iterable"),
{abstract_utils.T: field_types_union}, self.vm)
make = abstract.SimpleFunction(
name="_make",
param_names=("cls", "iterable", "new", "len"),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={
"new": self.vm.new_unsolvable(node),
"len": self.vm.new_unsolvable(node)
},
annotations={
"cls":
abstract.ParameterizedClass(self.vm.convert.type_type,
{abstract_utils.T: cls_type_param},
self.vm),
"iterable":
iterable_type,
"new":
self.vm.convert.unsolvable,
"len":
abstract.Callable(
self.vm.convert.name_to_value("typing.Callable"), {
0: sized_cls,
abstract_utils.ARGS: sized_cls,
abstract_utils.RET: self.vm.convert.int_type
}, self.vm),
"return":
cls_type_param
},
late_annotations={},
vm=self.vm).to_variable(node)
make_args = function.Args(posargs=(make, ))
_, members["_make"] = self.vm.special_builtins["classmethod"].call(
node, None, make_args)
# _replace
# Like __new__, it uses the _Tname TypeVar. We have to annotate the `self`
# param to make sure the TypeVar is substituted correctly.
members["_replace"] = abstract.SimpleFunction(
name="_replace",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name="kwds",
defaults={},
annotations={
"self": cls_type_param,
"kwds": field_types_union,
"return": cls_type_param
},
late_annotations={},
vm=self.vm).to_variable(node)
# __getnewargs__
getnewargs_tuple_params = dict(
tuple(enumerate(field_types)) +
((abstract_utils.T, field_types_union), ))
getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type,
getnewargs_tuple_params,
self.vm)
members["__getnewargs__"] = abstract.SimpleFunction(
name="__getnewargs__",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={
"return": getnewargs_tuple
},
late_annotations={},
vm=self.vm).to_variable(node)
# __getstate__
members["__getstate__"] = abstract.SimpleFunction(
name="__getstate__",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={},
late_annotations={},
vm=self.vm).to_variable(node)
# _asdict
members["_asdict"] = abstract.SimpleFunction(
name="_asdict",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={
"return": field_dict_cls
},
late_annotations={},
vm=self.vm).to_variable(node)
# Finally, make the class.
abs_membs = abstract.Dict(self.vm)
abs_membs.update(node, members)
if name.__class__ is compat.UnicodeType:
# Unicode values should be ASCII.
name = compat.native_str(name.encode("ascii"))
node, cls_var = self.vm.make_class(
node=node,
name_var=self.vm.convert.build_string(node, name),
bases=[self.vm.convert.tuple_type.to_variable(node)],
class_dict_var=abs_membs.to_variable(node),
cls_var=None)
# Now that the class has been made, we can complete the TypeParameter used
# by __new__, _make and _replace.
cls_type_param.bound = cls_var.data[0]
return node, cls_var
