def import_module_by_names(inference_state,
import_names,
sys_path=None,
module_context=None,
prefer_stubs=True):
if sys_path is None:
sys_path = inference_state.get_sys_path()
str_import_names = tuple(
force_unicode(i.value if isinstance(i, tree.Name) else i)
for i in import_names)
value_set = [None]
for i, name in enumerate(import_names):
value_set = ValueSet.from_sets([
import_module(
inference_state,
str_import_names[:i + 1],
parent_module_value,
sys_path,
prefer_stubs=prefer_stubs,
) for parent_module_value in value_set
])
if not value_set:
message = 'No module named ' + '.'.join(str_import_names)
if module_context is not None:
_add_error(module_context, name, message)
else:
debug.warning(message)
return NO_VALUES
return value_set
def _split_comment_param_declaration(decl_text):
"""
Split decl_text on commas, but group generic expressions
together.
For example, given "foo, Bar[baz, biz]" we return
['foo', 'Bar[baz, biz]'].
"""
try:
node = parse(decl_text, error_recovery=False).children[0]
except ParserSyntaxError:
debug.warning('Comment annotation is not valid Python: %s' % decl_text)
return []
if node.type in ['name', 'atom_expr', 'power']:
return [node.get_code().strip()]
params = []
try:
children = node.children
except AttributeError:
return []
else:
for child in children:
if child.type in ['name', 'atom_expr', 'power']:
params.append(child.get_code().strip())
return params
def _get_buildout_script_paths(search_path):
"""
if there is a 'buildout.cfg' file in one of the parent directories of the
given module it will return a list of all files in the buildout bin
directory that look like python files.
:param search_path: absolute path to the module.
:type search_path: str
"""
project_root = _get_parent_dir_with_file(search_path, 'buildout.cfg')
if not project_root:
return
bin_path = os.path.join(project_root, 'bin')
if not os.path.exists(bin_path):
return
for filename in os.listdir(bin_path):
try:
filepath = os.path.join(bin_path, filename)
with open(filepath, 'r') as f:
firstline = f.readline()
if firstline.startswith('#!') and 'python' in firstline:
yield filepath
except (UnicodeDecodeError, IOError) as e:
# Probably a binary file; permission error or race cond. because
# file got deleted. Ignore it.
debug.warning(unicode(e))
continue
def py__mro__(self):
mro = [self]
yield self
# TODO Do a proper mro resolution. Currently we are just listing
# classes. However, it's a complicated algorithm.
for lazy_cls in self.py__bases__():
# TODO there's multiple different mro paths possible if this yields
# multiple possibilities. Could be changed to be more correct.
for cls in lazy_cls.infer():
# TODO detect for TypeError: duplicate base class str,
# e.g. `class X(str, str): pass`
try:
mro_method = cls.py__mro__
except AttributeError:
# TODO add a TypeError like:
"""
>>> class Y(lambda: test): pass
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: function() argument 1 must be code, not str
>>> class Y(1): pass
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: int() takes at most 2 arguments (3 given)
"""
debug.warning('Super class of %s is not a class: %s', self,
cls)
else:
for cls_new in mro_method():
if cls_new not in mro:
mro.append(cls_new)
yield cls_new
def _iter_module_names(inference_state, paths):
# Python modules/packages
for path in paths:
try:
dirs = scandir(path)
except OSError:
# The file might not exist or reading it might lead to an error.
debug.warning("Not possible to list directory: %s", path)
continue
for dir_entry in dirs:
name = dir_entry.name
# First Namespaces then modules/stubs
if dir_entry.is_dir():
# pycache is obviously not an interestin namespace. Also the
# name must be a valid identifier.
# TODO use str.isidentifier, once Python 2 is removed
if name != '__pycache__' and not re.search(r'\W|^\d', name):
yield name
else:
if name.endswith('.pyi'): # Stub files
modname = name[:-4]
else:
modname = inspect.getmodulename(name)
if modname and '.' not in modname:
if modname != '__init__':
yield modname
def _get_classes(self):
if self._bound_lazy_value is not None:
return self._bound_lazy_value.infer()
if self._constraints_lazy_values:
return self.constraints
debug.warning('Tried to infer the TypeVar %s without a given type',
self._var_name)
return NO_VALUES
def _static_getmro_newstyle(klass):
mro = type.__dict__['__mro__'].__get__(klass)
if not isinstance(mro, (tuple, list)):
# There are unfortunately no tests for this, I was not able to
# reproduce this in pure Python. However should still solve the issue
# raised in GH #1517.
debug.warning('mro of %s returned %s, should be a tuple' %
(klass, mro))
return ()
return mro
def add(node_context, error_name, node, message=None, typ=Error, payload=None):
exception = CODES[error_name][1]
if _check_for_exception_catch(node_context, node, exception, payload):
return
# TODO this path is probably not right
module_context = node_context.get_root_context()
module_path = module_context.py__file__()
issue_instance = typ(error_name, module_path, node.start_pos, message)
debug.warning(str(issue_instance), format=False)
node_context.inference_state.analysis.append(issue_instance)
return issue_instance
def too_many_args(argument):
m = _error_argument_count(funcdef, len(unpacked_va))
# Just report an error for the first param that is not needed (like
# cPython).
if arguments.get_calling_nodes():
# There might not be a valid calling node so check for that first.
issues.append(
_add_argument_issue('type-error-too-many-arguments',
argument,
message=m))
else:
issues.append(None)
debug.warning('non-public warning: %s', m)
def _get_forward_reference_node(context, string):
try:
new_node = context.inference_state.grammar.parse(
force_unicode(string),
start_symbol='eval_input',
error_recovery=False)
except ParserSyntaxError:
debug.warning('Annotation not parsed: %s' % string)
return None
else:
module = context.tree_node.get_root_node()
parser_utils.move(new_node, module.end_pos[0])
new_node.parent = context.tree_node
return new_node
def py__call__(self, arguments):
unpacked = arguments.unpack()
key, lazy_value = next(unpacked, (None, None))
var_name = self._find_string_name(lazy_value)
# The name must be given, otherwise it's useless.
if var_name is None or key is not None:
debug.warning('Found a variable without a name %s', arguments)
return NO_VALUES
return ValueSet([
TypeVar.create_cached(self.inference_state, self.parent_context,
self._tree_name, var_name, unpacked)
])
def py__call__(self, arguments):
"""
def x() -> Callable[[Callable[..., _T]], _T]: ...
"""
# The 0th index are the arguments.
try:
param_values = self._generics_manager[0]
result_values = self._generics_manager[1]
except IndexError:
debug.warning('Callable[...] defined without two arguments')
return NO_VALUES
else:
from medi.inference.gradual.annotation import infer_return_for_callable
return infer_return_for_callable(arguments, param_values,
result_values)
def _apply_decorators(context, node):
"""
Returns the function, that should to be executed in the end.
This is also the places where the decorators are processed.
"""
if node.type == 'classdef':
decoratee_value = ClassValue(
context.inference_state,
parent_context=context,
tree_node=node
)
else:
decoratee_value = FunctionValue.from_context(context, node)
initial = values = ValueSet([decoratee_value])
if is_big_annoying_library(context):
return values
for dec in reversed(node.get_decorators()):
debug.dbg('decorator: %s %s', dec, values, color="MAGENTA")
with debug.increase_indent_cm():
dec_values = context.infer_node(dec.children[1])
trailer_nodes = dec.children[2:-1]
if trailer_nodes:
# Create a trailer and infer it.
trailer = tree.PythonNode('trailer', trailer_nodes)
trailer.parent = dec
dec_values = infer_trailer(context, dec_values, trailer)
if not len(dec_values):
code = dec.get_code(include_prefix=False)
# For the short future, we don't want to hear about the runtime
# decorator in typing that was intentionally omitted. This is not
# "correct", but helps with debugging.
if code != '@runtime\n':
debug.warning('decorator not found: %s on %s', dec, node)
return initial
values = dec_values.execute(arguments.ValuesArguments([values]))
if not len(values):
debug.warning('not possible to resolve wrappers found %s', node)
return initial
debug.dbg('decorator end %s', values, color="MAGENTA")
if values != initial:
return ValueSet([Decoratee(c, decoratee_value) for c in values])
return values
def execution_allowed(inference_state, node):
"""
A decorator to detect recursions in statements. In a recursion a statement
at the same place, in the same module may not be executed two times.
"""
pushed_nodes = inference_state.recursion_detector.pushed_nodes
if node in pushed_nodes:
debug.warning('catched stmt recursion: %s @%s', node,
getattr(node, 'start_pos', None))
yield False
else:
try:
pushed_nodes.append(node)
yield True
finally:
pushed_nodes.pop()
def _infer_param(function_value, param):
"""
Infers the type of a function parameter, using type annotations.
"""
annotation = param.annotation
if annotation is None:
# If no Python 3-style annotation, look for a Python 2-style comment
# annotation.
# Identify parameters to function in the same sequence as they would
# appear in a type comment.
all_params = [
child for child in param.parent.children if child.type == 'param'
]
node = param.parent.parent
comment = parser_utils.get_following_comment_same_line(node)
if comment is None:
return NO_VALUES
match = re.match(r"^#\s*type:\s*\(([^#]*)\)\s*->", comment)
if not match:
return NO_VALUES
params_comments = _split_comment_param_declaration(match.group(1))
# Find the specific param being investigated
index = all_params.index(param)
# If the number of parameters doesn't match length of type comment,
# ignore first parameter (assume it's self).
if len(params_comments) != len(all_params):
debug.warning("Comments length != Params length %s %s",
params_comments, all_params)
if function_value.is_bound_method():
if index == 0:
# Assume it's self, which is already handled
return NO_VALUES
index -= 1
if index >= len(params_comments):
return NO_VALUES
param_comment = params_comments[index]
return _infer_annotation_string(
function_value.get_default_param_context(), param_comment)
# Annotations are like default params and resolve in the same way.
context = function_value.get_default_param_context()
return infer_annotation(context, annotation)
def iterate():
for generator in self.execute_function_slots(iter_slot_names):
if generator.is_instance() and not generator.is_compiled():
# `__next__` logic.
if self.inference_state.environment.version_info.major == 2:
name = u'next'
else:
name = u'__next__'
next_slot_names = generator.get_function_slot_names(name)
if next_slot_names:
yield LazyKnownValues(
generator.execute_function_slots(next_slot_names))
else:
debug.warning(
'Instance has no __next__ function in %s.',
generator)
else:
for lazy_value in generator.py__iter__():
yield lazy_value
def wrapper(context, *args, **kwargs):
n = context.tree_node
inference_state = context.inference_state
try:
inference_state.inferred_element_counts[n] += 1
maximum = 300
if context.parent_context is None \
and context.get_value() is inference_state.builtins_module:
# Builtins should have a more generous inference limit.
# It is important that builtins can be executed, otherwise some
# functions that depend on certain builtins features would be
# broken, see e.g. GH #1432
maximum *= 100
if inference_state.inferred_element_counts[n] > maximum:
debug.warning('In value %s there were too many inferences.', n)
return NO_VALUES
except KeyError:
inference_state.inferred_element_counts[n] = 1
return func(context, *args, **kwargs)
def __init__(self, parent_context, tree_name, var_name, unpacked_args):
super(TypeVar, self).__init__(parent_context, tree_name)
self._var_name = var_name
self._constraints_lazy_values = []
self._bound_lazy_value = None
self._covariant_lazy_value = None
self._contravariant_lazy_value = None
for key, lazy_value in unpacked_args:
if key is None:
self._constraints_lazy_values.append(lazy_value)
else:
if key == 'bound':
self._bound_lazy_value = lazy_value
elif key == 'covariant':
self._covariant_lazy_value = lazy_value
elif key == 'contravariant':
self._contra_variant_lazy_value = lazy_value
else:
debug.warning('Invalid TypeVar param name %s', key)
def _get_paths_from_buildout_script(inference_state, buildout_script_path):
file_io = FileIO(buildout_script_path)
try:
module_node = inference_state.parse(
file_io=file_io,
cache=True,
cache_path=settings.cache_directory
)
except IOError:
debug.warning('Error trying to read buildout_script: %s', buildout_script_path)
return
from medi.inference.value import ModuleValue
module_context = ModuleValue(
inference_state, module_node,
file_io=file_io,
string_names=None,
code_lines=get_cached_code_lines(inference_state.grammar, buildout_script_path),
).as_context()
for path in check_sys_path_modifications(module_context):
yield path
def infer_annotation(context, annotation):
"""
Inferes an annotation node. This means that it inferes the part of
`int` here:
foo: int = 3
Also checks for forward references (strings)
"""
value_set = context.infer_node(annotation)
if len(value_set) != 1:
debug.warning("Inferred typing index %s should lead to 1 object, "
" not %s" % (annotation, value_set))
return value_set
inferred_value = list(value_set)[0]
if is_string(inferred_value):
result = _get_forward_reference_node(context,
inferred_value.get_safe_value())
if result is not None:
return context.infer_node(result)
return value_set
def _find_string_name(self, lazy_value):
if lazy_value is None:
return None
value_set = lazy_value.infer()
if not value_set:
return None
if len(value_set) > 1:
debug.warning('Found multiple values for a type variable: %s',
value_set)
name_value = next(iter(value_set))
try:
method = name_value.get_safe_value
except AttributeError:
return None
else:
safe_value = method(default=None)
if self.inference_state.environment.version_info.major == 2:
if isinstance(safe_value, bytes):
return force_unicode(safe_value)
if isinstance(safe_value, (str, unicode)):
return safe_value
return None
def push_execution(self, execution):
funcdef = execution.tree_node
# These two will be undone in pop_execution.
self._recursion_level += 1
self._parent_execution_funcs.append(funcdef)
module_context = execution.get_root_context()
if module_context.is_builtins_module():
# We have control over builtins so we know they are not recursing
# like crazy. Therefore we just let them execute always, because
# they usually just help a lot with getting good results.
return False
if self._recursion_level > recursion_limit:
debug.warning('Recursion limit (%s) reached', recursion_limit)
return True
if self._execution_count >= total_function_execution_limit:
debug.warning('Function execution limit (%s) reached', total_function_execution_limit)
return True
self._execution_count += 1
if self._funcdef_execution_counts.setdefault(funcdef, 0) >= per_function_execution_limit:
if module_context.py__name__() == 'typing':
return False
debug.warning(
'Per function execution limit (%s) reached: %s',
per_function_execution_limit,
funcdef
)
return True
self._funcdef_execution_counts[funcdef] += 1
if self._parent_execution_funcs.count(funcdef) > per_function_recursion_limit:
debug.warning(
'Per function recursion limit (%s) reached: %s',
per_function_recursion_limit,
funcdef
)
return True
return False
def iterate_argument_clinic(inference_state, arguments, clinic_string):
"""Uses a list with argument clinic information (see PEP 436)."""
clinic_args = list(_parse_argument_clinic(clinic_string))
iterator = PushBackIterator(arguments.unpack())
for i, (name, optional, allow_kwargs, stars) in enumerate(clinic_args):
if stars == 1:
lazy_values = []
for key, argument in iterator:
if key is not None:
iterator.push_back((key, argument))
break
lazy_values.append(argument)
yield ValueSet([iterable.FakeTuple(inference_state, lazy_values)])
lazy_values
continue
elif stars == 2:
raise NotImplementedError()
key, argument = next(iterator, (None, None))
if key is not None:
debug.warning('Keyword arguments in argument clinic are currently not supported.')
raise ParamIssue
if argument is None and not optional:
debug.warning('TypeError: %s expected at least %s arguments, got %s',
name, len(clinic_args), i)
raise ParamIssue
value_set = NO_VALUES if argument is None else argument.infer()
if not value_set and not optional:
# For the stdlib we always want values. If we don't get them,
# that's ok, maybe something is too hard to resolve, however,
# we will not proceed with the type inference of that function.
debug.warning('argument_clinic "%s" not resolvable.', name)
raise ParamIssue
yield value_set
def get_index_and_execute(self, index):
try:
return self[index].execute_annotation()
except IndexError:
debug.warning('No param #%s found for annotation %s', index, self)
return NO_VALUES
def execute_annotation(self):
debug.warning('Used Any - returned no results')
return NO_VALUES
def py__await__(self):
await_value_set = self.py__getattribute__(u"__await__")
if not await_value_set:
debug.warning('Tried to run __await__ on value %s', self)
return await_value_set.execute_with_values()
def py__get__(self, instance, class_value):
debug.warning("No __get__ defined on %s", self)
return ValueSet([self])
def py__stop_iteration_returns(self):
debug.warning("Not possible to return the stop iterations of %s", self)
return NO_VALUES
def py__call__(self, arguments):
debug.warning("no execution possible %s", self)
return NO_VALUES
def execute_operation(self, other, operator):
debug.warning("%s not possible between %s and %s", operator, self,
other)
return NO_VALUES
