def infer_return_types(function, arguments):
"""
Infers the type of a function's return value,
according to type annotations.
"""
all_annotations = py__annotations__(function.tree_node)
annotation = all_annotations.get("return", None)
if annotation is None:
# If there is no Python 3-type annotation, look for a Python 2-type annotation
node = function.tree_node
comment = parser_utils.get_following_comment_same_line(node)
if comment is None:
return NO_VALUES
match = re.match(r"^#\s*type:\s*\([^#]*\)\s*->\s*([^#]*)", comment)
if not match:
return NO_VALUES
return _infer_annotation_string(
function.get_default_param_context(),
match.group(1).strip()).execute_annotation()
context = function.get_default_param_context()
unknown_type_vars = find_unknown_type_vars(context, annotation)
annotation_values = infer_annotation(context, annotation)
if not unknown_type_vars:
return annotation_values.execute_annotation()
type_var_dict = infer_type_vars_for_execution(function, arguments,
all_annotations)
return ValueSet.from_sets(
ann.define_generics(type_var_dict) if isinstance(
ann, (DefineGenericBaseClass, TypeVar)) else ValueSet({ann})
for ann in annotation_values).execute_annotation()
def to_stub(value):
if value.is_stub():
return ValueSet([value])
was_instance = value.is_instance()
if was_instance:
value = value.py__class__()
qualified_names = value.get_qualified_names()
stub_module = _load_stub_module(value.get_root_context().get_value())
if stub_module is None or qualified_names is None:
return NO_VALUES
was_bound_method = value.is_bound_method()
if was_bound_method:
# Infer the object first. We can infer the method later.
method_name = qualified_names[-1]
qualified_names = qualified_names[:-1]
was_instance = True
stub_values = ValueSet([stub_module])
for name in qualified_names:
stub_values = stub_values.py__getattribute__(name)
if was_instance:
stub_values = ValueSet.from_sets(c.execute_with_values()
for c in stub_values if c.is_class())
if was_bound_method:
# Now that the instance has been properly created, we can simply get
# the method.
stub_values = stub_values.py__getattribute__(method_name)
return stub_values
def import_module_by_names(inference_state,
import_names,
sys_path=None,
module_context=None,
prefer_stubs=True):
if sys_path is None:
sys_path = inference_state.get_sys_path()
str_import_names = tuple(
force_unicode(i.value if isinstance(i, tree.Name) else i)
for i in import_names)
value_set = [None]
for i, name in enumerate(import_names):
value_set = ValueSet.from_sets([
import_module(
inference_state,
str_import_names[:i + 1],
parent_module_value,
sys_path,
prefer_stubs=prefer_stubs,
) for parent_module_value in value_set
])
if not value_set:
message = 'No module named ' + '.'.join(str_import_names)
if module_context is not None:
_add_error(module_context, name, message)
else:
debug.warning(message)
return NO_VALUES
return value_set
def convert_values(values,
only_stubs=False,
prefer_stubs=False,
ignore_compiled=True):
assert not (only_stubs and prefer_stubs)
with debug.increase_indent_cm('convert values'):
if only_stubs or prefer_stubs:
return ValueSet.from_sets(
to_stub(value) or (
ValueSet({value}) if prefer_stubs else NO_VALUES)
for value in values)
else:
return ValueSet.from_sets(
_stub_to_python_value_set(stub_value,
ignore_compiled=ignore_compiled)
or ValueSet({stub_value}) for stub_value in values)
def py__call__(self, arguments):
names = self.get_function_slot_names(u'__call__')
if not names:
# Means the Instance is not callable.
return super(_BaseTreeInstance, self).py__call__(arguments)
return ValueSet.from_sets(name.infer().execute(arguments)
for name in names)
def py__getattribute__(self,
name_or_str,
name_context=None,
position=None,
analysis_errors=True):
"""
:param position: Position of the last statement -> tuple of line, column
"""
if name_context is None:
name_context = self
names = self.goto(name_or_str, position)
string_name = name_or_str.value if isinstance(name_or_str,
Name) else name_or_str
# This paragraph is currently needed for proper branch type inference
# (static analysis).
found_predefined_types = None
if self.predefined_names and isinstance(name_or_str, Name):
node = name_or_str
while node is not None and not parser_utils.is_scope(node):
node = node.parent
if node.type in ("if_stmt", "for_stmt", "comp_for",
'sync_comp_for'):
try:
name_dict = self.predefined_names[node]
types = name_dict[string_name]
except KeyError:
continue
else:
found_predefined_types = types
break
if found_predefined_types is not None and names:
from medi.inference import flow_analysis
check = flow_analysis.reachability_check(
context=self,
value_scope=self.tree_node,
node=name_or_str,
)
if check is flow_analysis.UNREACHABLE:
values = NO_VALUES
else:
values = found_predefined_types
else:
values = ValueSet.from_sets(name.infer() for name in names)
if not names and not values and analysis_errors:
if isinstance(name_or_str, Name):
from medi.inference import analysis
message = ("NameError: name '%s' is not defined." %
string_name)
analysis.add(name_context, 'name-error', name_or_str, message)
debug.dbg('context.names_to_types: %s -> %s', names, values)
if values:
return values
return self._check_for_additional_knowledge(name_or_str, name_context,
position)
def py__getitem__(self, index_value_set, contextualized_node):
names = self.get_function_slot_names(u'__getitem__')
if not names:
return super(_BaseTreeInstance, self).py__getitem__(
index_value_set,
contextualized_node,
)
args = ValuesArguments([index_value_set])
return ValueSet.from_sets(name.infer().execute(args) for name in names)
def _execute_types_in_stmt(module_context, stmt):
"""
Executing all types or general elements that we find in a statement. This
doesn't include tuple, list and dict literals, because the stuff they
contain is executed. (Used as type information).
"""
definitions = module_context.infer_node(stmt)
return ValueSet.from_sets(
_execute_array_values(module_context.inference_state, d)
for d in definitions
)
def wrapper(inference_state, import_names, parent_module_value, sys_path,
prefer_stubs):
python_value_set = inference_state.module_cache.get(import_names)
if python_value_set is None:
if parent_module_value is not None and parent_module_value.is_stub(
):
parent_module_values = parent_module_value.non_stub_value_set
else:
parent_module_values = [parent_module_value]
if import_names == ('os', 'path'):
# This is a huge exception, we follow a nested import
# ``os.path``, because it's a very important one in Python
# that is being achieved by messing with ``sys.modules`` in
# ``os``.
python_value_set = ValueSet.from_sets(
func(
inference_state,
(n, ),
None,
sys_path,
) for n in
[u'posixpath', u'ntpath', u'macpath', u'os2emxpath'])
else:
python_value_set = ValueSet.from_sets(
func(
inference_state,
import_names,
p,
sys_path,
) for p in parent_module_values)
inference_state.module_cache.add(import_names, python_value_set)
if not prefer_stubs:
return python_value_set
stub = try_to_load_stub_cached(inference_state, import_names,
python_value_set, parent_module_value,
sys_path)
if stub is not None:
return ValueSet([stub])
return python_value_set
def infer_return_for_callable(arguments, param_values, result_values):
all_type_vars = {}
for pv in param_values:
if pv.array_type == 'list':
type_var_dict = _infer_type_vars_for_callable(
arguments, pv.py__iter__())
all_type_vars.update(type_var_dict)
return ValueSet.from_sets(
v.define_generics(all_type_vars) if isinstance(v, (
DefineGenericBaseClass, TypeVar)) else ValueSet({v})
for v in result_values).execute_annotation()
def _remap_type_vars(self, base):
from medi.inference.gradual.type_var import TypeVar
filter = self._class_value.get_type_var_filter()
for type_var_set in base.get_generics():
new = NO_VALUES
for type_var in type_var_set:
if isinstance(type_var, TypeVar):
names = filter.get(type_var.py__name__())
new |= ValueSet.from_sets(name.infer() for name in names)
else:
# Mostly will be type vars, except if in some cases
# a concrete type will already be there. In that
# case just add it to the value set.
new |= ValueSet([type_var])
yield new
def py__call__(self, arguments):
debug.dbg("Execute overloaded function %s",
self._wrapped_value,
color='BLUE')
function_executions = []
for signature in self.get_signatures():
function_execution = signature.value.as_context(arguments)
function_executions.append(function_execution)
if signature.matches_signature(arguments):
return function_execution.infer()
if self.inference_state.is_analysis:
# In this case we want precision.
return NO_VALUES
return ValueSet.from_sets(fe.infer() for fe in function_executions)
def get_key_values(self):
values = NO_VALUES
if self.array_type == 'dict':
for i, (key, instance) in enumerate(self._arguments.unpack()):
if key is None and i == 0:
values |= ValueSet.from_sets(v.get_key_values()
for v in instance.infer()
if v.array_type == 'dict')
if key:
values |= ValueSet([
compiled.create_simple_object(
self.inference_state,
key,
)
])
return values
def get_metaclasses(self):
args = self._get_bases_arguments()
if args is not None:
m = [value for key, value in args.unpack() if key == 'metaclass']
metaclasses = ValueSet.from_sets(lazy_value.infer()
for lazy_value in m)
metaclasses = ValueSet(m for m in metaclasses if m.is_class())
if metaclasses:
return metaclasses
for lazy_base in self.py__bases__():
for value in lazy_base.infer():
if value.is_class():
values = value.get_metaclasses()
if values:
return values
return NO_VALUES
def _execute_array_values(inference_state, array):
"""
Tuples indicate that there's not just one return value, but the listed
ones. `(str, int)` means that it returns a tuple with both types.
"""
from medi.inference.value.iterable import SequenceLiteralValue, FakeTuple, FakeList
if isinstance(array, SequenceLiteralValue) and array.array_type in ('tuple', 'list'):
values = []
for lazy_value in array.py__iter__():
objects = ValueSet.from_sets(
_execute_array_values(inference_state, typ)
for typ in lazy_value.infer()
)
values.append(LazyKnownValues(objects))
cls = FakeTuple if array.array_type == 'tuple' else FakeList
return {cls(inference_state, values)}
else:
return array.execute_annotation()
def _infer_comparison(context, left_values, operator, right_values):
state = context.inference_state
if not left_values or not right_values:
# illegal slices e.g. cause left/right_result to be None
result = (left_values or NO_VALUES) | (right_values or NO_VALUES)
return _literals_to_types(state, result)
else:
# I don't think there's a reasonable chance that a string
# operation is still correct, once we pass something like six
# objects.
if len(left_values) * len(right_values) > 6:
return _literals_to_types(state, left_values | right_values)
else:
return ValueSet.from_sets(
_infer_comparison_part(state, context, left, operator, right)
for left in left_values
for right in right_values
)
def dynamic_param_lookup(function_value, param_index):
"""
A dynamic search for param values. If you try to complete a type:
>>> def func(foo):
... foo
>>> func(1)
>>> func("")
It is not known what the type ``foo`` without analysing the whole code. You
have to look for all calls to ``func`` to find out what ``foo`` possibly
is.
"""
funcdef = function_value.tree_node
if not settings.dynamic_params:
return NO_VALUES
path = function_value.get_root_context().py__file__()
if path is not None and is_stdlib_path(path):
# We don't want to search for references in the stdlib. Usually people
# don't work with it (except if you are a core maintainer, sorry).
# This makes everything slower. Just disable it and run the tests,
# you will see the slowdown, especially in 3.6.
return NO_VALUES
if funcdef.type == 'lambdef':
string_name = _get_lambda_name(funcdef)
if string_name is None:
return NO_VALUES
else:
string_name = funcdef.name.value
debug.dbg('Dynamic param search in %s.', string_name, color='MAGENTA')
module_context = function_value.get_root_context()
arguments_list = _search_function_arguments(module_context, funcdef,
string_name)
values = ValueSet.from_sets(
get_executed_param_names(function_value, arguments)
[param_index].infer() for arguments in arguments_list)
debug.dbg('Dynamic param result finished', color='MAGENTA')
return values
def _infer(self, only_stubs=False, prefer_stubs=False):
assert not (only_stubs and prefer_stubs)
if not self._name.is_value_name:
return []
# First we need to make sure that we have stub names (if possible) that
# we can follow. If we don't do that, we can end up with the inferred
# results of Python objects instead of stubs.
names = convert_names([self._name], prefer_stubs=True)
values = convert_values(
ValueSet.from_sets(n.infer() for n in names),
only_stubs=only_stubs,
prefer_stubs=prefer_stubs,
)
resulting_names = [c.name for c in values]
return [
self if n == self._name else Name(self._inference_state, n)
for n in resulting_names
]
def py__simple_getitem__(self, index):
if self.array_type == 'dict':
# Logic for dict({'foo': bar}) and dict(foo=bar)
# reversed, because:
# >>> dict({'a': 1}, a=3)
# {'a': 3}
# TODO tuple initializations
# >>> dict([('a', 4)])
# {'a': 4}
for key, lazy_context in reversed(list(self._arguments.unpack())):
if key is None:
values = ValueSet.from_sets(
dct_value.py__simple_getitem__(index)
for dct_value in lazy_context.infer()
if dct_value.array_type == 'dict')
if values:
return values
else:
if key == index:
return lazy_context.infer()
return super(TreeInstance, self).py__simple_getitem__(index)
def get_return_values(self, check_yields=False):
funcdef = self.tree_node
if funcdef.type == 'lambdef':
return self.infer_node(funcdef.children[-1])
if check_yields:
value_set = NO_VALUES
returns = get_yield_exprs(self.inference_state, funcdef)
else:
value_set = self._infer_annotations()
if value_set:
# If there are annotations, prefer them over anything else.
# This will make it faster.
return value_set
value_set |= docstrings.infer_return_types(self._value)
returns = funcdef.iter_return_stmts()
for r in returns:
if check_yields:
value_set |= ValueSet.from_sets(
lazy_value.infer()
for lazy_value in self._get_yield_lazy_value(r))
else:
check = flow_analysis.reachability_check(self, funcdef, r)
if check is flow_analysis.UNREACHABLE:
debug.dbg('Return unreachable: %s', r)
else:
try:
children = r.children
except AttributeError:
ctx = compiled.builtin_from_name(
self.inference_state, u'None')
value_set |= ValueSet([ctx])
else:
value_set |= self.infer_node(children[1])
if check is flow_analysis.REACHABLE:
debug.dbg('Return reachable: %s', r)
break
return value_set
def execute_annotation(self):
if self.access_handle.get_repr() == 'None':
# None as an annotation doesn't need to be executed.
return ValueSet([self])
name, args = self.access_handle.get_annotation_name_and_args()
arguments = [
ValueSet([create_from_access_path(self.inference_state, path)])
for path in args
]
if name == 'Union':
return ValueSet.from_sets(arg.execute_annotation()
for arg in arguments)
elif name:
# While with_generics only exists on very specific objects, we
# should probably be fine, because we control all the typing
# objects.
return ValueSet([
v.with_generics(arguments) for v in
self.inference_state.typing_module.py__getattribute__(name)
]).execute_annotation()
return super(CompiledValue, self).execute_annotation()
def _stub_to_python_value_set(stub_value, ignore_compiled=False):
stub_module_context = stub_value.get_root_context()
if not stub_module_context.is_stub():
return ValueSet([stub_value])
decorates = None
if isinstance(stub_value, Decoratee):
decorates = stub_value._original_value
was_instance = stub_value.is_instance()
if was_instance:
stub_value = stub_value.py__class__()
qualified_names = stub_value.get_qualified_names()
if qualified_names is None:
return NO_VALUES
was_bound_method = stub_value.is_bound_method()
if was_bound_method:
# Infer the object first. We can infer the method later.
method_name = qualified_names[-1]
qualified_names = qualified_names[:-1]
was_instance = True
values = _infer_from_stub(stub_module_context, qualified_names,
ignore_compiled)
if was_instance:
values = ValueSet.from_sets(c.execute_with_values() for c in values
if c.is_class())
if was_bound_method:
# Now that the instance has been properly created, we can simply get
# the method.
values = values.py__getattribute__(method_name)
if decorates is not None:
values = ValueSet(Decoratee(v, decorates) for v in values)
return values
def infer(self):
return ValueSet.from_sets(l.infer() for l in self.data)
def execute_function_slots(self, names, *inferred_args):
return ValueSet.from_sets(
name.infer().execute_with_values(*inferred_args) for name in names)
def py__simple_getitem__(self, index):
if isinstance(index, unicode):
return ValueSet.from_sets(
name.infer() for filter in self._definition_class.get_filters(
is_instance=True) for name in filter.get(index))
return NO_VALUES
def py__getitem__(self, index_value_set, contextualized_node):
if self._is_homogenous():
return self._generics_manager.get_index_and_execute(0)
return ValueSet.from_sets(
self._generics_manager.to_tuple()).execute_annotation()
def gather_annotation_classes(self):
return ValueSet.from_sets(self._generics_manager.to_tuple())
def merge_yield_values(self, is_async=False):
return ValueSet.from_sets(
lazy_value.infer() for lazy_value in self.get_yield_lazy_values())
def tree_name_to_values(inference_state, context, tree_name):
value_set = NO_VALUES
module_node = context.get_root_context().tree_node
# First check for annotations, like: `foo: int = 3`
if module_node is not None:
names = module_node.get_used_names().get(tree_name.value, [])
found_annotation = False
for name in names:
expr_stmt = name.parent
if expr_stmt.type == "expr_stmt" and expr_stmt.children[1].type == "annassign":
correct_scope = parser_utils.get_parent_scope(name) == context.tree_node
if correct_scope:
found_annotation = True
value_set |= annotation.infer_annotation(
context, expr_stmt.children[1].children[1]
).execute_annotation()
if found_annotation:
return value_set
types = []
node = tree_name.get_definition(import_name_always=True, include_setitem=True)
if node is None:
node = tree_name.parent
if node.type == 'global_stmt':
c = context.create_context(tree_name)
if c.is_module():
# In case we are already part of the module, there is no point
# in looking up the global statement anymore, because it's not
# valid at that point anyway.
return NO_VALUES
# For global_stmt lookups, we only need the first possible scope,
# which means the function itself.
filter = next(c.get_filters())
names = filter.get(tree_name.value)
return ValueSet.from_sets(name.infer() for name in names)
elif node.type not in ('import_from', 'import_name'):
c = context.create_context(tree_name)
return infer_atom(c, tree_name)
typ = node.type
if typ == 'for_stmt':
types = annotation.find_type_from_comment_hint_for(context, node, tree_name)
if types:
return types
if typ == 'with_stmt':
types = annotation.find_type_from_comment_hint_with(context, node, tree_name)
if types:
return types
if typ in ('for_stmt', 'comp_for', 'sync_comp_for'):
try:
types = context.predefined_names[node][tree_name.value]
except KeyError:
cn = ContextualizedNode(context, node.children[3])
for_types = iterate_values(
cn.infer(),
contextualized_node=cn,
is_async=node.parent.type == 'async_stmt',
)
n = TreeNameDefinition(context, tree_name)
types = check_tuple_assignments(n, for_types)
elif typ == 'expr_stmt':
types = infer_expr_stmt(context, node, tree_name)
elif typ == 'with_stmt':
value_managers = context.infer_node(node.get_test_node_from_name(tree_name))
enter_methods = value_managers.py__getattribute__(u'__enter__')
return enter_methods.execute_with_values()
elif typ in ('import_from', 'import_name'):
types = imports.infer_import(context, tree_name)
elif typ in ('funcdef', 'classdef'):
types = _apply_decorators(context, node)
elif typ == 'try_stmt':
# TODO an exception can also be a tuple. Check for those.
# TODO check for types that are not classes and add it to
# the static analysis report.
exceptions = context.infer_node(tree_name.get_previous_sibling().get_previous_sibling())
types = exceptions.execute_with_values()
elif typ == 'param':
types = NO_VALUES
elif typ == 'del_stmt':
types = NO_VALUES
else:
raise ValueError("Should not happen. type: %s" % typ)
return types
def constraints(self):
return ValueSet.from_sets(lazy.infer()
for lazy in self._constraints_lazy_values)
