def _load_from_file_system(grammar, path, p_time):
cache_path = _get_hashed_path(grammar, path)
try:
try:
if p_time > os.path.getmtime(cache_path):
# Cache is outdated
return None
except OSError as e:
if e.errno == errno.ENOENT:
# In Python 2 instead of an IOError here we get an OSError.
raise FileNotFoundError
else:
raise
with open(cache_path, 'rb') as f:
gc.disable()
try:
module_cache_item = pickle.load(f)
finally:
gc.enable()
except FileNotFoundError:
return None
else:
parser_cache[path] = module_cache_item
debug.dbg('pickle loaded: %s', path)
return module_cache_item.node
def get_return_types(self, check_yields=False):
func = self.base
if func.isinstance(LambdaWrapper):
return self._evaluator.eval_element(self.children[-1])
if func.listeners:
# Feed the listeners, with the params.
for listener in func.listeners:
listener.execute(self._get_params())
# If we do have listeners, that means that there's not a regular
# execution ongoing. In this case Jedi is interested in the
# inserted params, not in the actual execution of the function.
return []
if check_yields:
types = []
returns = self.yields
else:
returns = self.returns
types = list(docstrings.find_return_types(self._evaluator, func))
for r in returns:
check = flow_analysis.break_check(self._evaluator, self, r)
if check is flow_analysis.UNREACHABLE:
debug.dbg('Return unreachable: %s', r)
else:
types += self._evaluator.eval_element(r.children[1])
if check is flow_analysis.REACHABLE:
debug.dbg('Return reachable: %s', r)
break
return types
def eval_trailer(context, base_contexts, trailer):
trailer_op, node = trailer.children[:2]
if node == ')': # `arglist` is optional.
node = None
if trailer_op == '[':
trailer_op, node, _ = trailer.children
# TODO It's kind of stupid to cast this from a context set to a set.
foo = set(base_contexts)
# special case: PEP0484 typing module, see
# https://github.com/davidhalter/jedi/issues/663
result = ContextSet()
for typ in list(foo):
if isinstance(typ, (ClassContext, TreeInstance)):
typing_module_types = pep0484.py__getitem__(context, typ, node)
if typing_module_types is not None:
foo.remove(typ)
result |= typing_module_types
return result | base_contexts.get_item(
eval_subscript_list(context.evaluator, context, node),
ContextualizedNode(context, trailer)
)
else:
debug.dbg('eval_trailer: %s in %s', trailer, base_contexts)
if trailer_op == '.':
return base_contexts.py__getattribute__(
name_context=context,
name_or_str=node
)
else:
assert trailer_op == '(', 'trailer_op is actually %s' % trailer_op
args = arguments.TreeArguments(context.evaluator, context, node, trailer)
return base_contexts.execute(args)
def find(self, scopes, attribute_lookup):
"""
:params bool attribute_lookup: Tell to logic if we're accessing the
attribute or the contents of e.g. a function.
"""
# TODO rename scopes to names_dicts
names = self.filter_name(scopes)
if self._found_predefined_if_name is not None:
return self._found_predefined_if_name
types = self._names_to_types(names, attribute_lookup)
if (
not names
and not types
and not (isinstance(self.name_str, tree.Name) and isinstance(self.name_str.parent.parent, tree.Param))
):
if not isinstance(self.name_str, (str, unicode)): # TODO Remove?
if attribute_lookup:
analysis.add_attribute_error(self._evaluator, self.scope, self.name_str)
else:
message = "NameError: name '%s' is not defined." % self.name_str
analysis.add(self._evaluator, "name-error", self.name_str, message)
debug.dbg("finder._names_to_types: %s -> %s", names, types)
return types
def get_return_values(self, check_yields=False):
funcdef = self.tree_node
if funcdef.type == 'lambdef':
return self.eval_node(funcdef.children[-1])
if check_yields:
context_set = NO_CONTEXTS
returns = get_yield_exprs(self.evaluator, funcdef)
else:
returns = funcdef.iter_return_stmts()
context_set = docstrings.infer_return_types(self.function_context)
context_set |= pep0484.infer_return_types(self.function_context)
for r in returns:
check = flow_analysis.reachability_check(self, funcdef, r)
if check is flow_analysis.UNREACHABLE:
debug.dbg('Return unreachable: %s', r)
else:
if check_yields:
context_set |= ContextSet.from_sets(
lazy_context.infer()
for lazy_context in self._get_yield_lazy_context(r)
)
else:
try:
children = r.children
except AttributeError:
ctx = compiled.builtin_from_name(self.evaluator, u'None')
context_set |= ContextSet(ctx)
else:
context_set |= self.eval_node(children[1])
if check is flow_analysis.REACHABLE:
debug.dbg('Return reachable: %s', r)
break
return context_set
def _copy_from_old_parser(self, line_offset, until_line_old, until_line_new):
copied_nodes = [None]
while until_line_new > self._nodes_stack.parsed_until_line:
parsed_until_line_old = self._nodes_stack.parsed_until_line - line_offset
line_stmt = self._get_old_line_stmt(parsed_until_line_old + 1)
if line_stmt is None:
# Parse 1 line at least. We don't need more, because we just
# want to get into a state where the old parser has statements
# again that can be copied (e.g. not lines within parentheses).
self._parse(self._nodes_stack.parsed_until_line + 1)
elif not copied_nodes:
# We have copied as much as possible (but definitely not too
# much). Therefore we just parse the rest.
# We might not reach the end, because there's a statement
# that is not finished.
self._parse(until_line_new)
else:
p_children = line_stmt.parent.children
index = p_children.index(line_stmt)
copied_nodes = self._nodes_stack.copy_nodes(
p_children[index:],
until_line_old,
line_offset
)
# Match all the nodes that are in the wanted range.
if copied_nodes:
self._copy_count += 1
from_ = copied_nodes[0].get_start_pos_of_prefix()[0] + line_offset
to = self._nodes_stack.parsed_until_line
self._copied_ranges.append((from_, to))
debug.dbg('diff actually copy %s to %s', from_, to)
def _apply_decorators(context, node):
"""
Returns the function, that should to be executed in the end.
This is also the places where the decorators are processed.
"""
if node.type == 'classdef':
decoratee_context = ClassContext(
context.evaluator,
parent_context=context,
tree_node=node
)
else:
decoratee_context = FunctionContext.from_context(context, node)
initial = values = ContextSet(decoratee_context)
for dec in reversed(node.get_decorators()):
debug.dbg('decorator: %s %s', dec, values)
dec_values = context.eval_node(dec.children[1])
trailer_nodes = dec.children[2:-1]
if trailer_nodes:
# Create a trailer and evaluate it.
trailer = tree.PythonNode('trailer', trailer_nodes)
trailer.parent = dec
dec_values = eval_trailer(context, dec_values, trailer)
if not len(dec_values):
debug.warning('decorator not found: %s on %s', dec, node)
return initial
values = dec_values.execute(arguments.ValuesArguments([values]))
if not len(values):
debug.warning('not possible to resolve wrappers found %s', node)
return initial
debug.dbg('decorator end %s', values)
return values
def get_return_values(self, check_yields=False):
funcdef = self.tree_node
if funcdef.type == 'lambdef':
return self.evaluator.eval_element(self, funcdef.children[-1])
if check_yields:
types = set()
returns = funcdef.iter_yield_exprs()
else:
returns = funcdef.iter_return_stmts()
types = set(docstrings.infer_return_types(self.function_context))
types |= set(pep0484.infer_return_types(self.function_context))
for r in returns:
check = flow_analysis.reachability_check(self, funcdef, r)
if check is flow_analysis.UNREACHABLE:
debug.dbg('Return unreachable: %s', r)
else:
if check_yields:
types |= set(self._eval_yield(r))
else:
types |= self.eval_node(r.children[1])
if check is flow_analysis.REACHABLE:
debug.dbg('Return reachable: %s', r)
break
return types
def _simple_complete(self, path, dot, like):
if not path and not dot:
scope = self._parser.user_scope()
if not scope.is_scope(): # Might be a flow (if/while/etc).
scope = scope.get_parent_scope()
names_dicts = global_names_dict_generator(
self._evaluator,
self._evaluator.wrap(scope),
self._pos
)
completion_names = []
for names_dict, pos in names_dicts:
names = list(chain.from_iterable(names_dict.values()))
if not names:
continue
completion_names += filter_definition_names(names, self._parser.user_stmt(), pos)
elif self._get_under_cursor_stmt(path) is None:
return []
else:
scopes = list(self._prepare_goto(path, True))
completion_names = []
debug.dbg('possible completion scopes: %s', scopes)
for s in scopes:
names = []
for names_dict in s.names_dicts(search_global=False):
names += chain.from_iterable(names_dict.values())
completion_names += filter_definition_names(names, self._parser.user_stmt())
return completion_names
def filter_name(self, filters):
"""
Searches names that are defined in a scope (the different
``filters``), until a name fits.
"""
names = []
if self._context.predefined_names:
# TODO is this ok? node might not always be a tree.Name
node = self._name
while node is not None and not is_scope(node):
node = node.parent
if node.type in ("if_stmt", "for_stmt", "comp_for"):
try:
name_dict = self._context.predefined_names[node]
types = name_dict[self._string_name]
except KeyError:
continue
else:
self._found_predefined_types = types
break
for filter in filters:
names = filter.get(self._string_name)
if names:
break
debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self._string_name,
self._context, names, self._position)
return list(names)
def __next__(self):
""" Generate the next tokenize pattern. """
try:
typ, tok, start_pos, end_pos, self.parserline = next(self._gen)
# dedents shouldn't change positions
if typ != tokenize.DEDENT:
self.start_pos, self.end_pos = start_pos, end_pos
except (StopIteration, common.MultiLevelStopIteration):
# on finish, set end_pos correctly
s = self.scope
while s is not None:
if isinstance(s, pr.Module) \
and not isinstance(s, pr.SubModule):
self.module.end_pos = self.end_pos
break
s.end_pos = self.end_pos
s = s.parent
raise
if self.user_position and (self.start_pos[0] == self.user_position[0]
or self.user_scope is None
and self.start_pos[0] >= self.user_position[0]):
debug.dbg('user scope found [%s] = %s' %
(self.parserline.replace('\n', ''), repr(self.scope)))
self.user_scope = self.scope
self.last_token = self.current
self.current = (typ, tok)
return self.current
def _names_to_types(self, names, attribute_lookup):
types = set()
types = unite(name.infer() for name in names)
debug.dbg('finder._names_to_types: %s -> %s', names, types)
if not names and isinstance(self._context, AbstractInstanceContext):
# handling __getattr__ / __getattribute__
return self._check_getattr(self._context)
# Add isinstance and other if/assert knowledge.
if not types and isinstance(self._name, tree.Name) and \
not isinstance(self._name_context, AbstractInstanceContext):
flow_scope = self._name
base_node = self._name_context.tree_node
if base_node.type == 'comp_for':
return types
while True:
flow_scope = get_parent_scope(flow_scope, include_flows=True)
n = _check_flow_information(self._name_context, flow_scope,
self._name, self._position)
if n is not None:
return n
if flow_scope == base_node:
break
return types
def search_params(evaluator, param):
"""
A dynamic search for param values. If you try to complete a type:
>>> def func(foo):
... foo
>>> func(1)
>>> func("")
It is not known what the type ``foo`` without analysing the whole code. You
have to look for all calls to ``func`` to find out what ``foo`` possibly
is.
"""
if not settings.dynamic_params:
return []
func = param.get_parent_until(tree.Function)
debug.dbg('Dynamic param search for %s in %s.', param, str(func.name))
# Compare the param names.
names = [n for n in search_function_call(evaluator, func)
if n.value == param.name.value]
# Evaluate the ExecutedParams to types.
result = list(chain.from_iterable(n.parent.eval(evaluator) for n in names))
debug.dbg('Dynamic param result %s', result)
return result
def _parse(self, source):
""" :type source: str """
added_newline = False
if not source or source[-1] != '\n':
# To be compatible with Pythons grammar, we need a newline at the
# end. The parser would handle it, but since the fast parser abuses
# the normal parser in various ways, we need to care for this
# ourselves.
source += '\n'
added_newline = True
next_line_offset = line_offset = 0
start = 0
nodes = list(self.current_node.all_sub_nodes())
# Now we can reset the node, because we have all the old nodes.
self.current_node.reset_node()
last_end_line = 1
for code_part in self._split_parts(source):
next_line_offset += code_part.count('\n')
# If the last code part parsed isn't equal to the current end_pos,
# we know that the parser went further (`def` start in a
# docstring). So just parse the next part.
if line_offset + 1 == last_end_line:
self.current_node = self._get_node(code_part, source[start:],
line_offset, nodes)
else:
# Means that some lines where not fully parsed. Parse it now.
# This is a very rare case. Should only happens with very
# strange code bits.
self.number_of_misses += 1
while last_end_line < next_line_offset + 1:
line_offset = last_end_line - 1
# We could calculate the src in a more complicated way to
# make caching here possible as well. However, this is
# complicated and error-prone. Since this is not very often
# called - just ignore it.
src = ''.join(self._lines[line_offset:])
self.current_node = self._get_node(code_part, src,
line_offset, nodes)
last_end_line = self.current_node.parser.module.end_pos[0]
debug.dbg('While parsing %s, line %s slowed down the fast parser.',
self.module_path, line_offset + 1)
line_offset = next_line_offset
start += len(code_part)
last_end_line = self.current_node.parser.module.end_pos[0]
if added_newline:
self.current_node.remove_last_newline()
# Now that the for loop is finished, we still want to close all nodes.
self.current_node = self.current_node.parent_until_indent()
self.current_node.close()
debug.dbg('Parsed %s, with %s parsers in %s splits.'
% (self.module_path, self.number_parsers_used,
self.number_of_splits))
def execute(self, obj, params=(), evaluate_generator=False):
if obj.isinstance(er.Function):
obj = obj.get_decorated_func()
debug.dbg('execute: %s %s', obj, params)
try:
return stdlib.execute(self, obj, params)
except stdlib.NotInStdLib:
pass
if isinstance(obj, iterable.GeneratorMethod):
return obj.execute()
elif obj.isinstance(compiled.CompiledObject):
if obj.is_executable_class():
return [er.Instance(self, obj, params)]
else:
return list(obj.execute_function(self, params))
elif obj.isinstance(er.Class):
# There maybe executions of executions.
return [er.Instance(self, obj, params)]
else:
stmts = []
if obj.isinstance(er.Function):
stmts = er.FunctionExecution(self, obj, params).get_return_types(evaluate_generator)
else:
if hasattr(obj, 'execute_subscope_by_name'):
try:
stmts = obj.execute_subscope_by_name('__call__', params)
except KeyError:
debug.warning("no __call__ func available %s", obj)
else:
debug.warning("no execution possible %s", obj)
debug.dbg('execute result: %s in %s', stmts, obj)
return imports.strip_imports(self, stmts)
def _simple_complete(self, path, like):
try:
scopes = list(self._prepare_goto(path, True))
except NotFoundError:
scopes = []
scope_generator = evaluate.get_names_of_scope(
self._parser.user_scope, self._pos)
completions = []
for scope, name_list in scope_generator:
for c in name_list:
completions.append((c, scope))
else:
completions = []
debug.dbg('possible scopes', scopes)
for s in scopes:
if s.isinstance(er.Function):
names = s.get_magic_method_names()
else:
if isinstance(s, imports.ImportPath):
under = like + self._module.get_path_after_cursor()
if under == 'import':
current_line = self._module.get_position_line()
if not current_line.endswith('import import'):
continue
a = s.import_stmt.alias
if a and a.start_pos <= self._pos <= a.end_pos:
continue
names = s.get_defined_names(on_import_stmt=True)
else:
names = s.get_defined_names()
for c in names:
completions.append((c, s))
return completions
def _prepare_goto(self, goto_path, is_completion=False):
"""
Base for completions/goto. Basically it returns the resolved scopes
under cursor.
"""
debug.dbg('start: %s in %s', goto_path, self._parser.user_scope())
user_stmt = self._parser.user_stmt_with_whitespace()
if not user_stmt and len(goto_path.split('\n')) > 1:
# If the user_stmt is not defined and the goto_path is multi line,
# something's strange. Most probably the backwards tokenizer
# matched to much.
return []
if isinstance(user_stmt, pr.Import):
i, _ = helpers.get_on_import_stmt(self._evaluator, self._user_context,
user_stmt, is_completion)
if i is None:
return []
scopes = [i]
else:
# just parse one statement, take it and evaluate it
eval_stmt = self._get_under_cursor_stmt(goto_path)
if eval_stmt is None:
return []
module = self._parser.module()
names, level, _, _ = helpers.check_error_statements(module, self._pos)
if names:
i = imports.get_importer(self._evaluator, names, module, level)
return i.follow(self._evaluator)
scopes = self._evaluator.eval_element(eval_stmt)
return scopes
def execute(self, obj, arguments=(), trailer=None):
if not isinstance(arguments, param.Arguments):
arguments = param.Arguments(self, arguments, trailer)
if self.is_analysis:
arguments.eval_all()
if obj.isinstance(er.Function):
obj = obj.get_decorated_func()
debug.dbg('execute: %s %s', obj, arguments)
try:
# Some stdlib functions like super(), namedtuple(), etc. have been
# hard-coded in Jedi to support them.
return stdlib.execute(self, obj, arguments)
except stdlib.NotInStdLib:
pass
try:
func = obj.py__call__
except AttributeError:
debug.warning("no execution possible %s", obj)
return set()
else:
types = func(arguments)
debug.dbg('execute result: %s in %s', types, obj)
return types
def _process(self):
debug.dbg('Start environment subprocess %s', self._executable)
parso_path = sys.modules['parso'].__file__
args = (
self._executable,
_MAIN_PATH,
os.path.dirname(os.path.dirname(parso_path)),
'.'.join(str(x) for x in sys.version_info[:3]),
)
process = GeneralizedPopen(
args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
# Use system default buffering on Python 2 to improve performance
# (this is already the case on Python 3).
bufsize=-1
)
self._stderr_queue = Queue()
self._stderr_thread = t = Thread(
target=_enqueue_output,
args=(process.stderr, self._stderr_queue)
)
t.daemon = True
t.start()
return process
def execute(self, arguments):
"""
In contrast to py__call__ this function is always available.
`hasattr(x, py__call__)` can also be checked to see if a context is
executable.
"""
if self.evaluator.is_analysis:
arguments.eval_all()
debug.dbg('execute: %s %s', self, arguments)
from jedi.evaluate import stdlib
try:
# Some stdlib functions like super(), namedtuple(), etc. have been
# hard-coded in Jedi to support them.
return stdlib.execute(self.evaluator, self, arguments)
except stdlib.NotInStdLib:
pass
try:
func = self.py__call__
except AttributeError:
debug.warning("no execution possible %s", self)
return NO_CONTEXTS
else:
context_set = func(arguments)
debug.dbg('execute result: %s in %s', context_set, self)
return context_set
return self.evaluator.execute(self, arguments)
def follow_str(ns_path, string):
debug.dbg('follow_module', ns_path, string)
path = None
if ns_path:
path = ns_path
elif self.import_stmt.relative_count:
path = self.get_relative_path()
global imports_processed
imports_processed += 1
importing = None
if path is not None:
importing = find_module(string, [path])
else:
debug.dbg('search_module', string, self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
sys.path, temp = sys_path_mod, sys.path
try:
importing = find_module(string)
except ImportError:
sys.path = temp
raise
sys.path = temp
return importing
def calculate_children(evaluator, children):
"""
Calculate a list of children with operators.
"""
iterator = iter(children)
types = evaluator.eval_element(next(iterator))
for operator in iterator:
right = next(iterator)
if tree.is_node(operator, 'comp_op'): # not in / is not
operator = ' '.join(str(c.value) for c in operator.children)
# handle lazy evaluation of and/or here.
if operator in ('and', 'or'):
left_bools = set([left.py__bool__() for left in types])
if left_bools == set([True]):
if operator == 'and':
types = evaluator.eval_element(right)
elif left_bools == set([False]):
if operator != 'and':
types = evaluator.eval_element(right)
# Otherwise continue, because of uncertainty.
else:
types = calculate(evaluator, types, operator,
evaluator.eval_element(right))
debug.dbg('calculate_children types %s', types)
return types
def eval_statement(self, stmt, seek_name=None):
"""
The starting point of the completion. A statement always owns a call
list, which are the calls, that a statement does. In case multiple
names are defined in the statement, `seek_name` returns the result for
this name.
:param stmt: A `pr.ExprStmt`.
"""
debug.dbg('eval_statement %s (%s)', stmt, seek_name)
types = self.eval_element(stmt.get_rhs())
if seek_name:
types = finder.check_tuple_assignments(types, seek_name)
first_operation = stmt.first_operation()
if first_operation not in ('=', None) and not isinstance(stmt, er.InstanceElement): # TODO don't check for this.
# `=` is always the last character in aug assignments -> -1
operator = copy.copy(first_operation)
operator.value = operator.value[:-1]
name = str(stmt.get_defined_names()[0])
parent = er.wrap(self, stmt.get_parent_scope())
left = self.find_types(parent, name, stmt.start_pos, search_global=True)
if isinstance(stmt.get_parent_until(pr.ForStmt), pr.ForStmt):
# Iterate through result and add the values, that's possible
# only in for loops without clutter, because they are
# predictable.
for r in types:
left = precedence.calculate(self, left, operator, [r])
types = left
else:
types = precedence.calculate(self, left, operator, types)
debug.dbg('eval_statement result %s', types)
return types
def _prepare_goto(self, goto_path, is_completion=False):
"""
Base for completions/goto. Basically it returns the resolved scopes
under cursor.
"""
debug.dbg('start: %s in %s', goto_path, self._parser.user_scope())
user_stmt = self._parser.user_stmt_with_whitespace()
if not user_stmt and len(goto_path.split('\n')) > 1:
# If the user_stmt is not defined and the goto_path is multi line,
# something's strange. Most probably the backwards tokenizer
# matched to much.
return []
if isinstance(user_stmt, pr.Import):
scopes = [helpers.get_on_import_stmt(self._evaluator, self._user_context,
user_stmt, is_completion)[0]]
else:
# just parse one statement, take it and evaluate it
eval_stmt = self._get_under_cursor_stmt(goto_path)
if not is_completion:
# goto_definition returns definitions of its statements if the
# cursor is on the assignee. By changing the start_pos of our
# "pseud" statement, the Jedi evaluator can find the assignees.
if user_stmt is not None:
eval_stmt.start_pos = user_stmt.end_pos
scopes = self._evaluator.eval_statement(eval_stmt)
return scopes
def get_return_types(self):
func = self.base
if func.listeners:
# Feed the listeners, with the params.
for listener in func.listeners:
listener.execute(self._get_params())
# If we do have listeners, that means that there's not a regular
# execution ongoing. In this case Jedi is interested in the
# inserted params, not in the actual execution of the function.
return []
types = list(docstrings.find_return_types(self._evaluator, func))
for r in self.returns:
if isinstance(r, pr.KeywordStatement):
stmt = r.stmt
else:
stmt = r # Lambdas
if stmt is None:
continue
check = flow_analysis.break_check(self._evaluator, self, r.parent)
if check is flow_analysis.UNREACHABLE:
debug.dbg('Return unreachable: %s', r)
else:
types += self._evaluator.eval_statement(stmt)
if check is flow_analysis.REACHABLE:
debug.dbg('Return reachable: %s', r)
break
return types
def _parse_function_doc(doc):
"""
Takes a function and returns the params and return value as a tuple.
This is nothing more than a docstring parser.
TODO docstrings like utime(path, (atime, mtime)) and a(b [, b]) -> None
TODO docstrings like 'tuple of integers'
"""
doc = force_unicode(doc)
# parse round parentheses: def func(a, (b,c))
try:
count = 0
start = doc.index('(')
for i, s in enumerate(doc[start:]):
if s == '(':
count += 1
elif s == ')':
count -= 1
if count == 0:
end = start + i
break
param_str = doc[start + 1:end]
except (ValueError, UnboundLocalError):
# ValueError for doc.index
# UnboundLocalError for undefined end in last line
debug.dbg('no brackets found - no param')
end = 0
param_str = u''
else:
# remove square brackets, that show an optional param ( = None)
def change_options(m):
args = m.group(1).split(',')
for i, a in enumerate(args):
if a and '=' not in a:
args[i] += '=None'
return ','.join(args)
while True:
param_str, changes = re.subn(r' ?\[([^\[\]]+)\]',
change_options, param_str)
if changes == 0:
break
param_str = param_str.replace('-', '_') # see: isinstance.__doc__
# parse return value
r = re.search(u'-[>-]* ', doc[end:end + 7])
if r is None:
ret = u''
else:
index = end + r.end()
# get result type, which can contain newlines
pattern = re.compile(r'(,\n|[^\n-])+')
ret_str = pattern.match(doc, index).group(0).strip()
# New object -> object()
ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
ret = docstr_defaults.get(ret_str, ret_str)
return param_str, ret
def _eval_element_not_cached(self, context, element):
debug.dbg('eval_element %s@%s', element, element.start_pos)
types = set()
typ = element.type
if typ in ('name', 'number', 'string', 'atom'):
types = self.eval_atom(context, element)
elif typ == 'keyword':
# For False/True/None
if element.value in ('False', 'True', 'None'):
types.add(compiled.builtin_from_name(self, element.value))
# else: print e.g. could be evaluated like this in Python 2.7
elif typ == 'lambdef':
types = set([er.FunctionContext(self, context, element)])
elif typ == 'expr_stmt':
types = self.eval_statement(context, element)
elif typ in ('power', 'atom_expr'):
first_child = element.children[0]
if not (first_child.type == 'keyword' and first_child.value == 'await'):
types = self.eval_atom(context, first_child)
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
right = self.eval_element(context, element.children[2])
types = set(precedence.calculate(self, context, types, trailer, right))
break
types = self.eval_trailer(context, types, trailer)
elif typ in ('testlist_star_expr', 'testlist',):
# The implicit tuple in statements.
types = set([iterable.SequenceLiteralContext(self, context, element)])
elif typ in ('not_test', 'factor'):
types = self.eval_element(context, element.children[-1])
for operator in element.children[:-1]:
types = set(precedence.factor_calculate(self, types, operator))
elif typ == 'test':
# `x if foo else y` case.
types = (self.eval_element(context, element.children[0]) |
self.eval_element(context, element.children[-1]))
elif typ == 'operator':
# Must be an ellipsis, other operators are not evaluated.
# In Python 2 ellipsis is coded as three single dot tokens, not
# as one token 3 dot token.
assert element.value in ('.', '...')
types = set([compiled.create(self, Ellipsis)])
elif typ == 'dotted_name':
types = self.eval_atom(context, element.children[0])
for next_name in element.children[2::2]:
# TODO add search_global=True?
types = unite(
typ.py__getattribute__(next_name, name_context=context)
for typ in types
)
types = types
elif typ == 'eval_input':
types = self._eval_element_not_cached(context, element.children[0])
elif typ == 'annassign':
types = pep0484._evaluate_for_annotation(context, element.children[1])
else:
types = precedence.calculate_children(self, context, element.children)
debug.dbg('eval_element result %s', types)
return types
def __call__(self, execution):
debug.dbg('Execution recursions: %s', execution, self.recursion_level,
self.execution_count, len(self.execution_funcs))
if self.check_recursion(execution):
result = set()
else:
result = self.func(execution)
self.pop_execution()
return result
def follow(self, is_goto=False):
if self._evaluator.recursion_detector.push_stmt(self.import_stmt):
# check recursion
return []
try:
if self.import_path:
try:
module, rest = self._importer.follow_file_system()
except ModuleNotFound as e:
analysis.add(self._evaluator, 'import-error', e.name_part)
return []
if module is None:
return []
if self.import_stmt.is_nested() and not self.nested_resolve:
scopes = [NestedImportModule(module, self.import_stmt)]
else:
scopes = [module]
star_imports = remove_star_imports(self._evaluator, module)
if star_imports:
scopes = [StarImportModule(scopes[0], star_imports)]
# goto only accepts `Name`
if is_goto and not rest:
scopes = [s.name for s in scopes]
# follow the rest of the import (not FS -> classes, functions)
if len(rest) > 1 or rest and self.is_like_search:
scopes = []
if ('os', 'path') == self.import_path[:2] \
and not self._is_relative_import():
# This is a huge exception, we follow a nested import
# ``os.path``, because it's a very important one in Python
# that is being achieved by messing with ``sys.modules`` in
# ``os``.
scopes = self._evaluator.follow_path(iter(rest), [module], module)
elif rest:
if is_goto:
scopes = list(chain.from_iterable(
self._evaluator.find_types(s, rest[0], is_goto=True)
for s in scopes))
else:
scopes = list(chain.from_iterable(
self._evaluator.follow_path(iter(rest), [s], s)
for s in scopes))
else:
scopes = [ImportWrapper.GlobalNamespace]
debug.dbg('after import: %s', scopes)
if not scopes:
analysis.add(self._evaluator, 'import-error',
self._importer.import_path[-1])
finally:
self._evaluator.recursion_detector.pop_stmt()
return scopes
def __call__(self, execution, evaluate_generator=False):
debug.dbg('Execution recursions: %s' % execution, self.recursion_level,
self.execution_count, len(self.execution_funcs))
if self.check_recursion(execution, evaluate_generator):
result = []
else:
result = self.func(execution, evaluate_generator)
self.cleanup()
return result
def _infer_node(context, element):
debug.dbg('infer_node %s@%s in %s', element, element.start_pos, context)
inference_state = context.inference_state
typ = element.type
if typ in ('name', 'number', 'string', 'atom', 'strings', 'keyword',
'fstring'):
return infer_atom(context, element)
elif typ == 'lambdef':
return ValueSet([FunctionValue.from_context(context, element)])
elif typ == 'expr_stmt':
return infer_expr_stmt(context, element)
elif typ in ('power', 'atom_expr'):
first_child = element.children[0]
children = element.children[1:]
had_await = False
if first_child.type == 'keyword' and first_child.value == 'await':
had_await = True
first_child = children.pop(0)
value_set = context.infer_node(first_child)
for (i, trailer) in enumerate(children):
if trailer == '**': # has a power operation.
right = context.infer_node(children[i + 1])
value_set = _infer_comparison(context, value_set, trailer,
right)
break
value_set = infer_trailer(context, value_set, trailer)
if had_await:
return value_set.py__await__().py__stop_iteration_returns()
return value_set
elif typ in (
'testlist_star_expr',
'testlist',
):
# The implicit tuple in statements.
return ValueSet(
[iterable.SequenceLiteralValue(inference_state, context, element)])
elif typ in ('not_test', 'factor'):
value_set = context.infer_node(element.children[-1])
for operator in element.children[:-1]:
value_set = infer_factor(value_set, operator)
return value_set
elif typ == 'test':
# `x if foo else y` case.
return (context.infer_node(element.children[0])
| context.infer_node(element.children[-1]))
elif typ == 'operator':
# Must be an ellipsis, other operators are not inferred.
# In Python 2 ellipsis is coded as three single dot tokens, not
# as one token 3 dot token.
if element.value not in ('.', '...'):
origin = element.parent
raise AssertionError("unhandled operator %s in %s " %
(repr(element.value), origin))
return ValueSet(
[compiled.builtin_from_name(inference_state, u'Ellipsis')])
elif typ == 'dotted_name':
value_set = infer_atom(context, element.children[0])
for next_name in element.children[2::2]:
value_set = value_set.py__getattribute__(next_name,
name_context=context)
return value_set
elif typ == 'eval_input':
return context.infer_node(element.children[0])
elif typ == 'annassign':
return annotation.infer_annotation(context, element.children[1]) \
.execute_annotation()
elif typ == 'yield_expr':
if len(element.children) and element.children[1].type == 'yield_arg':
# Implies that it's a yield from.
element = element.children[1].children[1]
generators = context.infer_node(element) \
.py__getattribute__('__iter__').execute_with_values()
return generators.py__stop_iteration_returns()
# Generator.send() is not implemented.
return NO_VALUES
elif typ == 'namedexpr_test':
return context.infer_node(element.children[2])
else:
return infer_or_test(context, element)
def _complete_trailer(self, previous_leaf):
inferred_context = self._module_context.create_context(previous_leaf)
values = infer_call_of_leaf(inferred_context, previous_leaf)
debug.dbg('trailer completion values: %s', values, color='MAGENTA')
return self._complete_trailer_for_values(values)
def test_simple():
jedi.set_debug_function()
debug.speed('foo')
debug.dbg('bar')
debug.warning('baz')
jedi.set_debug_function(None, False, False, False)
def _eval_element_not_cached(self, context, element):
debug.dbg('eval_element %s@%s', element, element.start_pos)
types = set()
typ = element.type
if typ in ('name', 'number', 'string', 'atom'):
types = self.eval_atom(context, element)
elif typ == 'keyword':
# For False/True/None
if element.value in ('False', 'True', 'None'):
types.add(compiled.builtin_from_name(self, element.value))
# else: print e.g. could be evaluated like this in Python 2.7
elif typ == 'lambda':
types = set([er.FunctionContext(self, context, element)])
elif typ == 'expr_stmt':
types = self.eval_statement(context, element)
elif typ in ('power', 'atom_expr'):
first_child = element.children[0]
if not (first_child.type == 'keyword'
and first_child.value == 'await'):
types = self.eval_atom(context, first_child)
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
right = self.eval_element(context, element.children[2])
types = set(
precedence.calculate(self, context, types, trailer,
right))
break
types = self.eval_trailer(context, types, trailer)
elif typ in (
'testlist_star_expr',
'testlist',
):
# The implicit tuple in statements.
types = set(
[iterable.SequenceLiteralContext(self, context, element)])
elif typ in ('not_test', 'factor'):
types = self.eval_element(context, element.children[-1])
for operator in element.children[:-1]:
types = set(precedence.factor_calculate(self, types, operator))
elif typ == 'test':
# `x if foo else y` case.
types = (self.eval_element(context, element.children[0])
| self.eval_element(context, element.children[-1]))
elif typ == 'operator':
# Must be an ellipsis, other operators are not evaluated.
assert element.value == '...'
types = set([compiled.create(self, Ellipsis)])
elif typ == 'dotted_name':
types = self.eval_atom(context, element.children[0])
for next_name in element.children[2::2]:
# TODO add search_global=True?
types = unite(
typ.py__getattribute__(next_name, name_context=context)
for typ in types)
types = types
elif typ == 'eval_input':
types = self._eval_element_not_cached(context, element.children[0])
elif typ == 'annassign':
types = pep0484._evaluate_for_annotation(context,
element.children[1])
else:
types = precedence.calculate_children(self, context,
element.children)
debug.dbg('eval_element result %s', types)
return types
def update(self, old_lines, new_lines):
'''
The algorithm works as follows:
Equal:
- Assure that the start is a newline, otherwise parse until we get
one.
- Copy from parsed_until_line + 1 to max(i2 + 1)
- Make sure that the indentation is correct (e.g. add DEDENT)
- Add old and change positions
Insert:
- Parse from parsed_until_line + 1 to min(j2 + 1), hopefully not
much more.
Returns the new module node.
'''
debug.speed('diff parser start')
# Reset the used names cache so they get regenerated.
self._module._used_names = None
self._parser_lines_new = new_lines
self._added_newline = False
if new_lines[-1] != '':
# The Python grammar needs a newline at the end of a file, but for
# everything else we keep working with new_lines here.
self._parser_lines_new = list(new_lines)
self._parser_lines_new[-1] += '\n'
self._parser_lines_new.append('')
self._added_newline = True
self._reset()
line_length = len(new_lines)
sm = difflib.SequenceMatcher(None, old_lines, self._parser_lines_new)
opcodes = sm.get_opcodes()
debug.speed('diff parser calculated')
debug.dbg('diff: line_lengths old: %s, new: %s' %
(len(old_lines), line_length))
for operation, i1, i2, j1, j2 in opcodes:
debug.dbg('diff %s old[%s:%s] new[%s:%s]', operation, i1 + 1, i2,
j1 + 1, j2)
if j2 == line_length + int(self._added_newline):
# The empty part after the last newline is not relevant.
j2 -= 1
if operation == 'equal':
line_offset = j1 - i1
self._copy_from_old_parser(line_offset, i2, j2)
elif operation == 'replace':
self._parse(until_line=j2)
elif operation == 'insert':
self._parse(until_line=j2)
else:
assert operation == 'delete'
# With this action all change will finally be applied and we have a
# changed module.
self._nodes_stack.close()
if self._added_newline:
_remove_last_newline(self._module)
# Good for debugging.
if debug.debug_function:
self._enabled_debugging(old_lines, new_lines)
last_pos = self._module.end_pos[0]
if last_pos != line_length:
current_lines = splitlines(self._module.get_code(), keepends=True)
diff = difflib.unified_diff(current_lines, new_lines)
raise Exception(
"There's an issue (%s != %s) with the diff parser. Please report:\n%s"
% (last_pos, line_length, ''.join(diff)))
debug.speed('diff parser end')
return self._module
def import_module(evaluator, import_names, parent_module_context, sys_path):
"""
This method is very similar to importlib's `_gcd_import`.
"""
if import_names[0] in settings.auto_import_modules:
module = _load_builtin_module(evaluator, import_names, sys_path)
if module is None:
return NO_CONTEXTS
return ContextSet([module])
module_name = '.'.join(import_names)
if parent_module_context is None:
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
string=import_names[-1],
full_name=module_name,
sys_path=sys_path,
is_global_search=True,
)
if is_pkg is None:
return NO_CONTEXTS
else:
try:
method = parent_module_context.py__path__
except AttributeError:
# The module is not a package.
return NO_CONTEXTS
else:
paths = method()
for path in paths:
# At the moment we are only using one path. So this is
# not important to be correct.
if not isinstance(path, list):
path = [path]
file_io_or_ns, is_pkg = evaluator.compiled_subprocess.get_module_info(
string=import_names[-1],
path=path,
full_name=module_name,
is_global_search=False,
)
if is_pkg is not None:
break
else:
return NO_CONTEXTS
if isinstance(file_io_or_ns, ImplicitNSInfo):
from jedi.evaluate.context.namespace import ImplicitNamespaceContext
module = ImplicitNamespaceContext(
evaluator,
fullname=file_io_or_ns.name,
paths=file_io_or_ns.paths,
)
elif file_io_or_ns is None:
module = _load_builtin_module(evaluator, import_names, sys_path)
if module is None:
return NO_CONTEXTS
else:
module = _load_python_module(
evaluator, file_io_or_ns, sys_path,
import_names=import_names,
is_package=is_pkg,
)
if parent_module_context is None:
debug.dbg('global search_module %s: %s', import_names[-1], module)
else:
debug.dbg('search_module %s in paths %s: %s', module_name, paths, module)
return ContextSet([module])
def _parse(self, source):
""" :type source: str """
added_newline = False
if not source or source[-1] != '\n':
# To be compatible with Pythons grammar, we need a newline at the
# end. The parser would handle it, but since the fast parser abuses
# the normal parser in various ways, we need to care for this
# ourselves.
source += '\n'
added_newline = True
next_line_offset = line_offset = 0
start = 0
nodes = list(self.current_node.all_sub_nodes())
# Now we can reset the node, because we have all the old nodes.
self.current_node.reset_node()
last_end_line = 1
for code_part in self._split_parts(source):
next_line_offset += code_part.count('\n')
# If the last code part parsed isn't equal to the current end_pos,
# we know that the parser went further (`def` start in a
# docstring). So just parse the next part.
if line_offset + 1 == last_end_line:
self.current_node = self._get_node(code_part, source[start:],
line_offset, nodes)
else:
# Means that some lines where not fully parsed. Parse it now.
# This is a very rare case. Should only happens with very
# strange code bits.
self.number_of_misses += 1
while last_end_line < next_line_offset + 1:
line_offset = last_end_line - 1
# We could calculate the src in a more complicated way to
# make caching here possible as well. However, this is
# complicated and error-prone. Since this is not very often
# called - just ignore it.
src = ''.join(self._lines[line_offset:])
self.current_node = self._get_node(code_part, src,
line_offset, nodes)
last_end_line = self.current_node.parser.module.end_pos[0]
debug.dbg(
'While parsing %s, line %s slowed down the fast parser.',
self.module_path, line_offset + 1)
line_offset = next_line_offset
start += len(code_part)
last_end_line = self.current_node.parser.module.end_pos[0]
if added_newline:
self.current_node.remove_last_newline()
# Now that the for loop is finished, we still want to close all nodes.
self.current_node = self.current_node.parent_until_indent()
self.current_node.close()
debug.dbg('Parsed %s, with %s parsers in %s splits.' %
(self.module_path, self.number_parsers_used,
self.number_of_splits))
def goto(self, name_or_str, name_context=None, analysis_errors=True):
from jedi.inference import finder
filters = self._get_value_filters(name_or_str)
names = finder.filter_name(filters, name_or_str)
debug.dbg('context.goto %s in (%s): %s', name_or_str, self, names)
return names
def find(self, scopes, resolve_decorator=True):
names = self.filter_name(scopes)
types = self._names_to_types(names, resolve_decorator)
debug.dbg('finder._names_to_types: %s, old: %s', names, types)
return self._resolve_descriptors(types)
def eval_node(context, element):
debug.dbg('eval_node %s@%s', element, element.start_pos)
evaluator = context.evaluator
typ = element.type
if typ in ('name', 'number', 'string', 'atom', 'strings', 'keyword'):
return eval_atom(context, element)
elif typ == 'lambdef':
return ContextSet(FunctionContext(evaluator, context, element))
elif typ == 'expr_stmt':
return eval_expr_stmt(context, element)
elif typ in ('power', 'atom_expr'):
first_child = element.children[0]
children = element.children[1:]
had_await = False
if first_child.type == 'keyword' and first_child.value == 'await':
had_await = True
first_child = children.pop(0)
context_set = eval_atom(context, first_child)
for trailer in children:
if trailer == '**': # has a power operation.
right = context.eval_node(children[1])
context_set = _eval_comparison(evaluator, context, context_set,
trailer, right)
break
context_set = eval_trailer(context, context_set, trailer)
if had_await:
await_context_set = context_set.py__getattribute__(u"__await__")
if not await_context_set:
debug.warning('Tried to run py__await__ on context %s',
context)
context_set = ContextSet()
return _py__stop_iteration_returns(
await_context_set.execute_evaluated())
return context_set
elif typ in (
'testlist_star_expr',
'testlist',
):
# The implicit tuple in statements.
return ContextSet(
iterable.SequenceLiteralContext(evaluator, context, element))
elif typ in ('not_test', 'factor'):
context_set = context.eval_node(element.children[-1])
for operator in element.children[:-1]:
context_set = eval_factor(context_set, operator)
return context_set
elif typ == 'test':
# `x if foo else y` case.
return (context.eval_node(element.children[0])
| context.eval_node(element.children[-1]))
elif typ == 'operator':
# Must be an ellipsis, other operators are not evaluated.
# In Python 2 ellipsis is coded as three single dot tokens, not
# as one token 3 dot token.
if element.value not in ('.', '...'):
origin = element.parent
raise AssertionError("unhandled operator %s in %s " %
(repr(element.value), origin))
return ContextSet(compiled.builtin_from_name(evaluator, u'Ellipsis'))
elif typ == 'dotted_name':
context_set = eval_atom(context, element.children[0])
for next_name in element.children[2::2]:
# TODO add search_global=True?
context_set = context_set.py__getattribute__(next_name,
name_context=context)
return context_set
elif typ == 'eval_input':
return eval_node(context, element.children[0])
elif typ == 'annassign':
return pep0484._evaluate_for_annotation(context, element.children[1])
elif typ == 'yield_expr':
if len(element.children) and element.children[1].type == 'yield_arg':
# Implies that it's a yield from.
element = element.children[1].children[1]
generators = context.eval_node(element)
return _py__stop_iteration_returns(generators)
# Generator.send() is not implemented.
return NO_CONTEXTS
else:
return eval_or_test(context, element)
def get_metaclass_filters(self, metaclass):
debug.dbg('Unprocessed metaclass %s', metaclass)
return []
def eval_element(self, context, element):
if isinstance(context, CompForContext):
return eval_node(context, element)
if_stmt = element
while if_stmt is not None:
if_stmt = if_stmt.parent
if if_stmt.type in ('if_stmt', 'for_stmt'):
break
if parser_utils.is_scope(if_stmt):
if_stmt = None
break
predefined_if_name_dict = context.predefined_names.get(if_stmt)
# TODO there's a lot of issues with this one. We actually should do
# this in a different way. Caching should only be active in certain
# cases and this all sucks.
if predefined_if_name_dict is None and if_stmt \
and if_stmt.type == 'if_stmt' and self.is_analysis:
if_stmt_test = if_stmt.children[1]
name_dicts = [{}]
# If we already did a check, we don't want to do it again -> If
# context.predefined_names is filled, we stop.
# We don't want to check the if stmt itself, it's just about
# the content.
if element.start_pos > if_stmt_test.end_pos:
# Now we need to check if the names in the if_stmt match the
# names in the suite.
if_names = helpers.get_names_of_node(if_stmt_test)
element_names = helpers.get_names_of_node(element)
str_element_names = [e.value for e in element_names]
if any(i.value in str_element_names for i in if_names):
for if_name in if_names:
definitions = self.goto_definitions(context, if_name)
# Every name that has multiple different definitions
# causes the complexity to rise. The complexity should
# never fall below 1.
if len(definitions) > 1:
if len(name_dicts) * len(definitions) > 16:
debug.dbg(
'Too many options for if branch evaluation %s.',
if_stmt)
# There's only a certain amount of branches
# Jedi can evaluate, otherwise it will take to
# long.
name_dicts = [{}]
break
original_name_dicts = list(name_dicts)
name_dicts = []
for definition in definitions:
new_name_dicts = list(original_name_dicts)
for i, name_dict in enumerate(new_name_dicts):
new_name_dicts[i] = name_dict.copy()
new_name_dicts[i][
if_name.value] = ContextSet(definition)
name_dicts += new_name_dicts
else:
for name_dict in name_dicts:
name_dict[if_name.value] = definitions
if len(name_dicts) > 1:
result = ContextSet()
for name_dict in name_dicts:
with helpers.predefine_names(context, if_stmt, name_dict):
result |= eval_node(context, element)
return result
else:
return self._eval_element_if_evaluated(context, element)
else:
if predefined_if_name_dict:
return eval_node(context, element)
else:
return self._eval_element_if_evaluated(context, element)
def eval_node(context, element):
debug.dbg('eval_element %s@%s', element, element.start_pos)
evaluator = context.evaluator
typ = element.type
if typ in ('name', 'number', 'string', 'atom'):
return eval_atom(context, element)
elif typ == 'keyword':
# For False/True/None
if element.value in ('False', 'True', 'None'):
return ContextSet(
compiled.builtin_from_name(evaluator, element.value))
# else: print e.g. could be evaluated like this in Python 2.7
return NO_CONTEXTS
elif typ == 'lambdef':
return ContextSet(FunctionContext(evaluator, context, element))
elif typ == 'expr_stmt':
return eval_expr_stmt(context, element)
elif typ in ('power', 'atom_expr'):
first_child = element.children[0]
if not (first_child.type == 'keyword'
and first_child.value == 'await'):
context_set = eval_atom(context, first_child)
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
right = evaluator.eval_element(context,
element.children[2])
context_set = _eval_comparison(evaluator, context,
context_set, trailer, right)
break
context_set = eval_trailer(context, context_set, trailer)
return context_set
return NO_CONTEXTS
elif typ in (
'testlist_star_expr',
'testlist',
):
# The implicit tuple in statements.
return ContextSet(
iterable.SequenceLiteralContext(evaluator, context, element))
elif typ in ('not_test', 'factor'):
context_set = context.eval_node(element.children[-1])
for operator in element.children[:-1]:
context_set = eval_factor(context_set, operator)
return context_set
elif typ == 'test':
# `x if foo else y` case.
return (context.eval_node(element.children[0])
| context.eval_node(element.children[-1]))
elif typ == 'operator':
# Must be an ellipsis, other operators are not evaluated.
# In Python 2 ellipsis is coded as three single dot tokens, not
# as one token 3 dot token.
assert element.value in ('.',
'...'), 'value is actually ' + element.value
return ContextSet(compiled.create(evaluator, Ellipsis))
elif typ == 'dotted_name':
context_set = eval_atom(context, element.children[0])
for next_name in element.children[2::2]:
# TODO add search_global=True?
context_set = context_set.py__getattribute__(next_name,
name_context=context)
return context_set
elif typ == 'eval_input':
return eval_node(context, element.children[0])
elif typ == 'annassign':
return pep0484._evaluate_for_annotation(context, element.children[1])
else:
return eval_or_test(context, element)
def _infer_comparison_part(inference_state, context, left, operator, right):
l_is_num = is_number(left)
r_is_num = is_number(right)
if isinstance(operator, str):
str_operator = operator
else:
str_operator = str(operator.value)
if str_operator == '*':
# for iterables, ignore * operations
if isinstance(left, iterable.Sequence) or is_string(left):
return ValueSet([left])
elif isinstance(right, iterable.Sequence) or is_string(right):
return ValueSet([right])
elif str_operator == '+':
if l_is_num and r_is_num or is_string(left) and is_string(right):
return left.execute_operation(right, str_operator)
elif _is_list(left) and _is_list(right) or _is_tuple(
left) and _is_tuple(right):
return ValueSet(
[iterable.MergedArray(inference_state, (left, right))])
elif str_operator == '-':
if l_is_num and r_is_num:
return left.execute_operation(right, str_operator)
elif str_operator == '%':
# With strings and numbers the left type typically remains. Except for
# `int() % float()`.
return ValueSet([left])
elif str_operator in COMPARISON_OPERATORS:
if left.is_compiled() and right.is_compiled():
# Possible, because the return is not an option. Just compare.
result = left.execute_operation(right, str_operator)
if result:
return result
else:
if str_operator in ('is', '!=', '==', 'is not'):
operation = COMPARISON_OPERATORS[str_operator]
bool_ = operation(left, right)
# Only if == returns True or != returns False, we can continue.
# There's no guarantee that they are not equal. This can help
# in some cases, but does not cover everything.
if (str_operator in ('is', '==')) == bool_:
return ValueSet([_bool_to_value(inference_state, bool_)])
if isinstance(left, VersionInfo):
version_info = _get_tuple_ints(right)
if version_info is not None:
bool_result = compiled.access.COMPARISON_OPERATORS[
operator](inference_state.environment.version_info,
tuple(version_info))
return ValueSet(
[_bool_to_value(inference_state, bool_result)])
return ValueSet([
_bool_to_value(inference_state, True),
_bool_to_value(inference_state, False)
])
elif str_operator in ('in', 'not in'):
return NO_VALUES
def check(obj):
"""Checks if a Jedi object is either a float or an int."""
return isinstance(obj, TreeInstance) and \
obj.name.string_name in ('int', 'float')
# Static analysis, one is a number, the other one is not.
if str_operator in ('+', '-') and l_is_num != r_is_num \
and not (check(left) or check(right)):
message = "TypeError: unsupported operand type(s) for +: %s and %s"
analysis.add(context, 'type-error-operation', operator,
message % (left, right))
if left.is_class() or right.is_class():
return NO_VALUES
method_name = operator_to_magic_method[str_operator]
magic_methods = left.py__getattribute__(method_name)
if magic_methods:
result = magic_methods.execute_with_values(right)
if result:
return result
if not magic_methods:
reverse_method_name = reverse_operator_to_magic_method[str_operator]
magic_methods = right.py__getattribute__(reverse_method_name)
result = magic_methods.execute_with_values(left)
if result:
return result
result = ValueSet([left, right])
debug.dbg('Used operator %s resulting in %s', operator, result)
return result
def _search_func(self, string, complete=False, all_scopes=False):
# Using a Script is they easiest way to get an empty module context.
from jedi import Script
s = Script('', project=self)
inference_state = s._inference_state
empty_module_context = s._get_module_context()
debug.dbg('Search for string %s, complete=%s', string, complete)
wanted_type, wanted_names = split_search_string(string)
name = wanted_names[0]
stub_folder_name = name + '-stubs'
ios = recurse_find_python_folders_and_files(FolderIO(str(self._path)))
file_ios = []
# 1. Search for modules in the current project
for folder_io, file_io in ios:
if file_io is None:
file_name = folder_io.get_base_name()
if file_name == name or file_name == stub_folder_name:
f = folder_io.get_file_io('__init__.py')
try:
m = load_module_from_path(inference_state,
f).as_context()
except FileNotFoundError:
f = folder_io.get_file_io('__init__.pyi')
try:
m = load_module_from_path(inference_state,
f).as_context()
except FileNotFoundError:
m = load_namespace_from_path(
inference_state, folder_io).as_context()
else:
continue
else:
file_ios.append(file_io)
if Path(file_io.path).name in (name + '.py', name + '.pyi'):
m = load_module_from_path(inference_state,
file_io).as_context()
else:
continue
debug.dbg('Search of a specific module %s', m)
yield from search_in_module(
inference_state,
m,
names=[m.name],
wanted_type=wanted_type,
wanted_names=wanted_names,
complete=complete,
convert=True,
ignore_imports=True,
)
# 2. Search for identifiers in the project.
for module_context in search_in_file_ios(inference_state, file_ios,
name):
names = get_module_names(module_context.tree_node,
all_scopes=all_scopes)
names = [module_context.create_name(n) for n in names]
names = _remove_imports(names)
yield from search_in_module(
inference_state,
module_context,
names=names,
wanted_type=wanted_type,
wanted_names=wanted_names,
complete=complete,
ignore_imports=True,
)
# 3. Search for modules on sys.path
sys_path = [
p for p in self._get_sys_path(inference_state)
# Exclude folders that are handled by recursing of the Python
# folders.
if not p.startswith(str(self._path))
]
names = list(
iter_module_names(inference_state, empty_module_context, sys_path))
yield from search_in_module(
inference_state,
empty_module_context,
names=names,
wanted_type=wanted_type,
wanted_names=wanted_names,
complete=complete,
convert=True,
)
def get_return_types(self, evaluate_generator=False):
""" Get the return types of a function. """
base = self.decorated
stmts = []
if base.parent == builtin.Builtin.scope \
and not isinstance(base, (Generator, Array)):
func_name = str(base.name)
# some implementations of builtins:
if func_name == 'getattr':
# follow the first param
objects = self.follow_var_arg(0)
names = self.follow_var_arg(1)
for obj in objects:
if not isinstance(obj, (Instance, Class, pr.Module)):
debug.warning('getattr called without instance')
continue
for arr_name in names:
if not isinstance(arr_name, Instance):
debug.warning('getattr called without str')
continue
if len(arr_name.var_args) != 1:
debug.warning('jedi getattr is too simple')
key = arr_name.var_args[0]
stmts += evaluate.follow_path(iter([key]), obj, base)
return stmts
elif func_name == 'type':
# otherwise it would be a metaclass
if len(self.var_args) == 1:
objects = self.follow_var_arg(0)
return [o.base for o in objects if isinstance(o, Instance)]
elif func_name == 'super':
# TODO make this able to detect multiple inheritance supers
accept = (pr.Function, )
func = self.var_args.get_parent_until(accept)
if func.isinstance(*accept):
cls = func.get_parent_until(accept + (pr.Class, ),
include_current=False)
if isinstance(cls, pr.Class):
cls = Class(cls)
su = cls.get_super_classes()
if su:
return [Instance(su[0])]
return []
if base.isinstance(Class):
# There maybe executions of executions.
return [Instance(base, self.var_args)]
elif isinstance(base, Generator):
return base.iter_content()
else:
try:
base.returns # Test if it is a function
except AttributeError:
if hasattr(base, 'execute_subscope_by_name'):
try:
stmts = base.execute_subscope_by_name(
'__call__', self.var_args)
except KeyError:
debug.warning("no __call__ func available", base)
else:
debug.warning("no execution possible", base)
else:
stmts = self._get_function_returns(base, evaluate_generator)
debug.dbg('exec result: %s in %s' % (stmts, self))
return imports.strip_imports(stmts)
def execute(value, arguments):
debug.dbg('execute: %s %s', value, arguments)
with debug.increase_indent_cm():
value_set = value.py__call__(arguments=arguments)
debug.dbg('execute result: %s in %s', value_set, value)
return value_set
def eval_element(self, element):
if isinstance(element, iterable.AlreadyEvaluated):
return set(element)
elif isinstance(element, iterable.MergedNodes):
return iterable.unite(self.eval_element(e) for e in element)
if_stmt = element.get_parent_until(
(tree.IfStmt, tree.ForStmt, tree.IsScope))
predefined_if_name_dict = self.predefined_if_name_dict_dict.get(
if_stmt)
if predefined_if_name_dict is None and isinstance(
if_stmt, tree.IfStmt):
if_stmt_test = if_stmt.children[1]
name_dicts = [{}]
# If we already did a check, we don't want to do it again -> If
# predefined_if_name_dict_dict is filled, we stop.
# We don't want to check the if stmt itself, it's just about
# the content.
if element.start_pos > if_stmt_test.end_pos:
# Now we need to check if the names in the if_stmt match the
# names in the suite.
if_names = helpers.get_names_of_node(if_stmt_test)
element_names = helpers.get_names_of_node(element)
str_element_names = [str(e) for e in element_names]
if any(str(i) in str_element_names for i in if_names):
for if_name in if_names:
definitions = self.goto_definitions(if_name)
# Every name that has multiple different definitions
# causes the complexity to rise. The complexity should
# never fall below 1.
if len(definitions) > 1:
if len(name_dicts) * len(definitions) > 16:
debug.dbg(
'Too many options for if branch evaluation %s.',
if_stmt)
# There's only a certain amount of branches
# Jedi can evaluate, otherwise it will take to
# long.
name_dicts = [{}]
break
original_name_dicts = list(name_dicts)
name_dicts = []
for definition in definitions:
new_name_dicts = list(original_name_dicts)
for i, name_dict in enumerate(new_name_dicts):
new_name_dicts[i] = name_dict.copy()
new_name_dicts[i][str(if_name)] = [
definition
]
name_dicts += new_name_dicts
else:
for name_dict in name_dicts:
name_dict[str(if_name)] = definitions
if len(name_dicts) > 1:
result = set()
for name_dict in name_dicts:
self.predefined_if_name_dict_dict[if_stmt] = name_dict
try:
result |= self._eval_element_not_cached(element)
finally:
del self.predefined_if_name_dict_dict[if_stmt]
return result
else:
return self._eval_element_if_evaluated(element)
return self._eval_element_cached(element)
else:
if predefined_if_name_dict:
return self._eval_element_not_cached(element)
else:
return self._eval_element_if_evaluated(element)
return self._eval_element_cached(element)
def _execute(context, arguments):
debug.dbg('execute: %s %s', context, arguments)
with debug.increase_indent_cm():
context_set = context.py__call__(arguments=arguments)
debug.dbg('execute result: %s in %s', context_set, context)
return context_set
def _check_array_additions(evaluator, compare_array, module, is_list):
"""
Checks if a `Array` has "add" (append, insert, extend) statements:
>>> a = [""]
>>> a.append(1)
"""
debug.dbg('Dynamic array search for %s' % compare_array, color='MAGENTA')
if not settings.dynamic_array_additions or isinstance(
module, compiled.CompiledObject):
debug.dbg('Dynamic array search aborted.', color='MAGENTA')
return set()
def check_additions(arglist, add_name):
params = list(param.Arguments(evaluator, arglist).unpack())
result = set()
if add_name in ['insert']:
params = params[1:]
if add_name in ['append', 'add', 'insert']:
for key, nodes in params:
result |= unite(evaluator.eval_element(node) for node in nodes)
elif add_name in ['extend', 'update']:
for key, nodes in params:
for node in nodes:
types = evaluator.eval_element(node)
result |= py__iter__types(evaluator, types, node)
return result
from jedi.evaluate import representation as er, param
def get_execution_parent(element):
""" Used to get an Instance/FunctionExecution parent """
if isinstance(element, Array):
node = element.atom
else:
# Is an Instance with an
# Arguments([AlreadyEvaluated([_ArrayInstance])]) inside
# Yeah... I know... It's complicated ;-)
node = list(element.var_args.argument_node[0])[0].var_args.trailer
if isinstance(node, er.InstanceElement) or node is None:
return node
return node.get_parent_until(er.FunctionExecution)
temp_param_add, settings.dynamic_params_for_other_modules = \
settings.dynamic_params_for_other_modules, False
search_names = ['append', 'extend', 'insert'
] if is_list else ['add', 'update']
comp_arr_parent = get_execution_parent(compare_array)
added_types = set()
for add_name in search_names:
try:
possible_names = module.used_names[add_name]
except KeyError:
continue
else:
for name in possible_names:
# Check if the original scope is an execution. If it is, one
# can search for the same statement, that is in the module
# dict. Executions are somewhat special in jedi, since they
# literally copy the contents of a function.
if isinstance(comp_arr_parent, er.FunctionExecution):
if comp_arr_parent.start_pos < name.start_pos < comp_arr_parent.end_pos:
name = comp_arr_parent.name_for_position(
name.start_pos)
else:
# Don't check definitions that are not defined in the
# same function. This is not "proper" anyway. It also
# improves Jedi's speed for array lookups, since we
# don't have to check the whole source tree anymore.
continue
trailer = name.parent
power = trailer.parent
trailer_pos = power.children.index(trailer)
try:
execution_trailer = power.children[trailer_pos + 1]
except IndexError:
continue
else:
if execution_trailer.type != 'trailer' \
or execution_trailer.children[0] != '(' \
or execution_trailer.children[1] == ')':
continue
power = helpers.call_of_leaf(name, cut_own_trailer=True)
# InstanceElements are special, because they don't get copied,
# but have this wrapper around them.
if isinstance(comp_arr_parent, er.InstanceElement):
power = er.get_instance_el(evaluator,
comp_arr_parent.instance, power)
if evaluator.recursion_detector.push_stmt(power):
# Check for recursion. Possible by using 'extend' in
# combination with function calls.
continue
try:
if compare_array in evaluator.eval_element(power):
# The arrays match. Now add the results
added_types |= check_additions(
execution_trailer.children[1], add_name)
finally:
evaluator.recursion_detector.pop_stmt()
# reset settings
settings.dynamic_params_for_other_modules = temp_param_add
debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA')
return added_types
def _parse_function_doc(func):
"""
Takes a function and returns the params and return value as a tuple.
This is nothing more than a docstring parser.
"""
# TODO: things like utime(path, (atime, mtime)) and a(b [, b]) -> None
doc = inspect.getdoc(func)
if doc is None:
return '', 'pass'
# get full string, parse round parentheses: def func(a, (b,c))
try:
count = 0
debug.dbg(func, func.__name__, doc)
start = doc.index('(')
for i, s in enumerate(doc[start:]):
if s == '(':
count += 1
elif s == ')':
count -= 1
if count == 0:
end = start + i
break
param_str = doc[start + 1:end]
except (ValueError, UnboundLocalError, AttributeError):
# ValueError for doc.index
# UnboundLocalError for undefined end in last line
debug.dbg('no brackets found - no param')
end = 0
param_str = ''
else:
# remove square brackets, that show an optional param ( = None)
def change_options(m):
args = m.group(1).split(',')
for i, a in enumerate(args):
if a and '=' not in a:
args[i] += '=None'
return ','.join(args)
while True:
param_str, changes = re.subn(r' ?\[([^\[\]]+)\]', change_options,
param_str)
if changes == 0:
break
param_str = param_str.replace('-', '_') # see: isinstance.__doc__
# parse return value
r = re.search('-[>-]* ', doc[end:end + 7])
if r is None:
ret = ''
else:
index = end + r.end()
# get result type, which can contain newlines
pattern = re.compile(r'(,\n|[^\n-])+')
ret_str = pattern.match(doc, index).group(0).strip()
# New object -> object()
ret_str = re.sub(r'[nN]ew (.*)', r'\1()', ret_str)
ret = BuiltinModule.map_types.get(ret_str, ret_str)
if ret == ret_str and ret not in ['None', 'object', 'tuple', 'set']:
debug.dbg('not working', ret_str)
ret = ('return ' if 'return' not in ret else '') + ret
return param_str, ret
def _do_import(self, import_path, sys_path):
"""
This method is very similar to importlib's `_gcd_import`.
"""
import_parts = [str(i) for i in import_path]
# Handle "magic" Flask extension imports:
# ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``.
if len(import_path) > 2 and import_parts[:2] == ['flask', 'ext']:
# New style.
ipath = ('flask_' + str(import_parts[2]), ) + import_path[3:]
modules = self._do_import(ipath, sys_path)
if modules:
return modules
else:
# Old style
return self._do_import(('flaskext', ) + import_path[2:],
sys_path)
module_name = '.'.join(import_parts)
try:
return set([self._evaluator.modules[module_name]])
except KeyError:
pass
if len(import_path) > 1:
# This is a recursive way of importing that works great with
# the module cache.
bases = self._do_import(import_path[:-1], sys_path)
if not bases:
return set()
# We can take the first element, because only the os special
# case yields multiple modules, which is not important for
# further imports.
parent_module = list(bases)[0]
# This is a huge exception, we follow a nested import
# ``os.path``, because it's a very important one in Python
# that is being achieved by messing with ``sys.modules`` in
# ``os``.
if [str(i) for i in import_path] == ['os', 'path']:
return parent_module.py__getattribute__('path')
try:
method = parent_module.py__path__
except AttributeError:
# The module is not a package.
_add_error(parent_module, import_path[-1])
return set()
else:
paths = method()
debug.dbg('search_module %s in paths %s', module_name, paths)
for path in paths:
# At the moment we are only using one path. So this is
# not important to be correct.
try:
if not isinstance(path, list):
path = [path]
module_file, module_path, is_pkg = \
find_module(import_parts[-1], path, fullname=module_name)
break
except ImportError:
module_path = None
if module_path is None:
_add_error(parent_module, import_path[-1])
return set()
else:
parent_module = None
try:
debug.dbg('search_module %s in %s', import_parts[-1],
self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
sys.path, temp = sys_path, sys.path
try:
module_file, module_path, is_pkg = \
find_module(import_parts[-1], fullname=module_name)
finally:
sys.path = temp
except ImportError:
# The module is not a package.
_add_error(self.module_context, import_path[-1])
return set()
source = None
if is_pkg:
# In this case, we don't have a file yet. Search for the
# __init__ file.
if module_path.endswith(('.zip', '.egg')):
source = module_file.loader.get_source(module_name)
else:
module_path = get_init_path(module_path)
elif module_file:
source = module_file.read()
module_file.close()
if isinstance(module_path, ImplicitNSInfo):
from jedi.evaluate.representation import ImplicitNamespaceContext
fullname, paths = module_path.name, module_path.paths
module = ImplicitNamespaceContext(self._evaluator,
fullname=fullname)
module.paths = paths
elif module_file is None and not module_path.endswith(
('.py', '.zip', '.egg')):
module = compiled.load_module(self._evaluator, module_path)
else:
module = _load_module(self._evaluator, module_path, source,
sys_path, parent_module)
if module is None:
# The file might raise an ImportError e.g. and therefore not be
# importable.
return set()
self._evaluator.modules[module_name] = module
return set([module])
def _do_import(self, import_path, sys_path):
"""
This method is very similar to importlib's `_gcd_import`.
"""
import_parts = [str(i) for i in import_path]
# Handle "magic" Flask extension imports:
# ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``.
if len(import_path) > 2 and import_parts[:2] == ['flask', 'ext']:
# New style.
ipath = ('flask_' + str(import_parts[2]),) + import_path[3:]
modules = self._do_import(ipath, sys_path)
if modules:
return modules
else:
# Old style
return self._do_import(('flaskext',) + import_path[2:], sys_path)
module_name = '.'.join(import_parts)
try:
return set([self._evaluator.modules[module_name]])
except KeyError:
pass
if len(import_path) > 1:
# This is a recursive way of importing that works great with
# the module cache.
bases = self._do_import(import_path[:-1], sys_path)
if not bases:
return set()
# We can take the first element, because only the os special
# case yields multiple modules, which is not important for
# further imports.
base = list(bases)[0]
# This is a huge exception, we follow a nested import
# ``os.path``, because it's a very important one in Python
# that is being achieved by messing with ``sys.modules`` in
# ``os``.
if [str(i) for i in import_path] == ['os', 'path']:
return self._evaluator.find_types(base, 'path')
try:
# It's possible that by giving it always the sys path (and not
# the __path__ attribute of the parent, we get wrong results
# and nested namespace packages don't work. But I'm not sure.
paths = base.py__path__(sys_path)
except AttributeError:
# The module is not a package.
_add_error(self._evaluator, import_path[-1])
return set()
else:
debug.dbg('search_module %s in paths %s', module_name, paths)
for path in paths:
# At the moment we are only using one path. So this is
# not important to be correct.
try:
module_file, module_path, is_pkg = \
find_module(import_parts[-1], [path])
break
except ImportError:
module_path = None
if module_path is None:
_add_error(self._evaluator, import_path[-1])
return set()
else:
try:
debug.dbg('search_module %s in %s', import_parts[-1], self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
sys.path, temp = sys_path, sys.path
try:
module_file, module_path, is_pkg = \
find_module(import_parts[-1])
finally:
sys.path = temp
except ImportError:
# The module is not a package.
_add_error(self._evaluator, import_path[-1])
return set()
source = None
if is_pkg:
# In this case, we don't have a file yet. Search for the
# __init__ file.
module_path = get_init_path(module_path)
elif module_file:
source = module_file.read()
module_file.close()
if module_file is None and not module_path.endswith('.py'):
module = compiled.load_module(self._evaluator, module_path)
else:
module = _load_module(self._evaluator, module_path, source, sys_path)
self._evaluator.modules[module_name] = module
return set([module])
def import_module(inference_state, import_names, parent_module_value,
sys_path):
"""
This method is very similar to importlib's `_gcd_import`.
"""
if import_names[0] in settings.auto_import_modules:
module = _load_builtin_module(inference_state, import_names, sys_path)
if module is None:
return NO_VALUES
return ValueSet([module])
module_name = '.'.join(import_names)
if parent_module_value is None:
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
string=import_names[-1],
full_name=module_name,
sys_path=sys_path,
is_global_search=True,
)
if is_pkg is None:
return NO_VALUES
else:
paths = parent_module_value.py__path__()
if paths is None:
# The module might not be a package.
return NO_VALUES
for path in paths:
# At the moment we are only using one path. So this is
# not important to be correct.
if not isinstance(path, list):
path = [path]
file_io_or_ns, is_pkg = inference_state.compiled_subprocess.get_module_info(
string=import_names[-1],
path=path,
full_name=module_name,
is_global_search=False,
)
if is_pkg is not None:
break
else:
return NO_VALUES
if isinstance(file_io_or_ns, ImplicitNSInfo):
from jedi.inference.value.namespace import ImplicitNamespaceValue
module = ImplicitNamespaceValue(
inference_state,
string_names=tuple(file_io_or_ns.name.split('.')),
paths=file_io_or_ns.paths,
)
elif file_io_or_ns is None:
module = _load_builtin_module(inference_state, import_names, sys_path)
if module is None:
return NO_VALUES
else:
module = _load_python_module(
inference_state,
file_io_or_ns,
import_names=import_names,
is_package=is_pkg,
)
if parent_module_value is None:
debug.dbg('global search_module %s: %s', import_names[-1], module)
else:
debug.dbg('search_module %s in paths %s: %s', module_name, paths,
module)
return ValueSet([module])
def initialize(self, code):
debug.dbg('differ: initialize', color='YELLOW')
self.lines = splitlines(code, keepends=True)
parser_cache.pop(None, None)
self.module = parse(code, diff_cache=True, cache=True)
return self.module
def _check_array_additions(context, sequence):
"""
Checks if a `Array` has "add" (append, insert, extend) statements:
>>> a = [""]
>>> a.append(1)
"""
from jedi.evaluate import arguments
debug.dbg('Dynamic array search for %s' % sequence, color='MAGENTA')
module_context = context.get_root_context()
if not settings.dynamic_array_additions or isinstance(
module_context, compiled.CompiledObject):
debug.dbg('Dynamic array search aborted.', color='MAGENTA')
return ContextSet()
def find_additions(context, arglist, add_name):
params = list(
arguments.TreeArguments(context.evaluator, context,
arglist).unpack())
result = set()
if add_name in ['insert']:
params = params[1:]
if add_name in ['append', 'add', 'insert']:
for key, whatever in params:
result.add(whatever)
elif add_name in ['extend', 'update']:
for key, lazy_context in params:
result |= set(lazy_context.infer().iterate())
return result
temp_param_add, settings.dynamic_params_for_other_modules = \
settings.dynamic_params_for_other_modules, False
is_list = sequence.name.string_name == 'list'
search_names = (['append', 'extend', 'insert']
if is_list else ['add', 'update'])
added_types = set()
for add_name in search_names:
try:
possible_names = module_context.tree_node.get_used_names(
)[add_name]
except KeyError:
continue
else:
for name in possible_names:
context_node = context.tree_node
if not (context_node.start_pos < name.start_pos <
context_node.end_pos):
continue
trailer = name.parent
power = trailer.parent
trailer_pos = power.children.index(trailer)
try:
execution_trailer = power.children[trailer_pos + 1]
except IndexError:
continue
else:
if execution_trailer.type != 'trailer' \
or execution_trailer.children[0] != '(' \
or execution_trailer.children[1] == ')':
continue
random_context = context.create_context(name)
with recursion.execution_allowed(context.evaluator,
power) as allowed:
if allowed:
found = evaluate_call_of_leaf(random_context,
name,
cut_own_trailer=True)
if sequence in found:
# The arrays match. Now add the results
added_types |= find_additions(
random_context, execution_trailer.children[1],
add_name)
# reset settings
settings.dynamic_params_for_other_modules = temp_param_add
debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA')
return added_types
def _infer_comparison_part(inference_state, context, left, operator, right):
l_is_num = is_number(left)
r_is_num = is_number(right)
if isinstance(operator, unicode):
str_operator = operator
else:
str_operator = force_unicode(str(operator.value))
if str_operator == '*':
# for iterables, ignore * operations
if isinstance(left, iterable.Sequence) or is_string(left):
return ValueSet([left])
elif isinstance(right, iterable.Sequence) or is_string(right):
return ValueSet([right])
elif str_operator == '+':
if l_is_num and r_is_num or is_string(left) and is_string(right):
return ValueSet([left.execute_operation(right, str_operator)])
elif _is_tuple(left) and _is_tuple(right) or _is_list(
left) and _is_list(right):
return ValueSet(
[iterable.MergedArray(inference_state, (left, right))])
elif str_operator == '-':
if l_is_num and r_is_num:
return ValueSet([left.execute_operation(right, str_operator)])
elif str_operator == '%':
# With strings and numbers the left type typically remains. Except for
# `int() % float()`.
return ValueSet([left])
elif str_operator in COMPARISON_OPERATORS:
if left.is_compiled() and right.is_compiled():
# Possible, because the return is not an option. Just compare.
try:
return ValueSet([left.execute_operation(right, str_operator)])
except TypeError:
# Could be True or False.
pass
else:
if str_operator in ('is', '!=', '==', 'is not'):
operation = COMPARISON_OPERATORS[str_operator]
bool_ = operation(left, right)
return ValueSet([_bool_to_value(inference_state, bool_)])
if isinstance(left, VersionInfo):
version_info = _get_tuple_ints(right)
if version_info is not None:
bool_result = compiled.access.COMPARISON_OPERATORS[
operator](inference_state.environment.version_info,
tuple(version_info))
return ValueSet(
[_bool_to_value(inference_state, bool_result)])
return ValueSet([
_bool_to_value(inference_state, True),
_bool_to_value(inference_state, False)
])
elif str_operator == 'in':
return NO_VALUES
def check(obj):
"""Checks if a Jedi object is either a float or an int."""
return isinstance(obj, TreeInstance) and \
obj.name.string_name in ('int', 'float')
# Static analysis, one is a number, the other one is not.
if str_operator in ('+', '-') and l_is_num != r_is_num \
and not (check(left) or check(right)):
message = "TypeError: unsupported operand type(s) for +: %s and %s"
analysis.add(context, 'type-error-operation', operator,
message % (left, right))
result = ValueSet([left, right])
debug.dbg('Used operator %s resulting in %s', operator, result)
return result
def _infer_expr_stmt(context, stmt, seek_name=None):
"""
The starting point of the completion. A statement always owns a call
list, which are the calls, that a statement does. In case multiple
names are defined in the statement, `seek_name` returns the result for
this name.
expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
('=' (yield_expr|testlist_star_expr))*)
annassign: ':' test ['=' test]
augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')
:param stmt: A `tree.ExprStmt`.
"""
def check_setitem(stmt):
atom_expr = stmt.children[0]
if atom_expr.type not in ('atom_expr', 'power'):
return False, None
name = atom_expr.children[0]
if name.type != 'name' or len(atom_expr.children) != 2:
return False, None
trailer = atom_expr.children[-1]
return trailer.children[0] == '[', trailer.children[1]
debug.dbg('infer_expr_stmt %s (%s)', stmt, seek_name)
rhs = stmt.get_rhs()
value_set = context.infer_node(rhs)
if seek_name:
n = TreeNameDefinition(context, seek_name)
value_set = check_tuple_assignments(n, value_set)
first_operator = next(stmt.yield_operators(), None)
is_setitem, subscriptlist = check_setitem(stmt)
is_annassign = first_operator not in (
'=', None) and first_operator.type == 'operator'
if is_annassign or is_setitem:
# `=` is always the last character in aug assignments -> -1
name = stmt.get_defined_names(include_setitem=True)[0].value
left_values = context.py__getattribute__(name, position=stmt.start_pos)
if is_setitem:
def to_mod(v):
c = ContextualizedSubscriptListNode(context, subscriptlist)
if v.array_type == 'dict':
return DictModification(v, value_set, c)
elif v.array_type == 'list':
return ListModification(v, value_set, c)
return v
value_set = ValueSet(to_mod(v) for v in left_values)
else:
operator = copy.copy(first_operator)
operator.value = operator.value[:-1]
for_stmt = tree.search_ancestor(stmt, 'for_stmt')
if for_stmt is not None and for_stmt.type == 'for_stmt' and value_set \
and parser_utils.for_stmt_defines_one_name(for_stmt):
# Iterate through result and add the values, that's possible
# only in for loops without clutter, because they are
# predictable. Also only do it, if the variable is not a tuple.
node = for_stmt.get_testlist()
cn = ContextualizedNode(context, node)
ordered = list(cn.infer().iterate(cn))
for lazy_value in ordered:
dct = {for_stmt.children[1].value: lazy_value.infer()}
with context.predefine_names(for_stmt, dct):
t = context.infer_node(rhs)
left_values = _infer_comparison(
context, left_values, operator, t)
value_set = left_values
else:
value_set = _infer_comparison(context, left_values, operator,
value_set)
debug.dbg('infer_expr_stmt result %s', value_set)
return value_set
def _do_import(self, import_path, sys_path):
"""
This method is very similar to importlib's `_gcd_import`.
"""
import_parts = [
force_unicode(i.value if isinstance(i, tree.Name) else i)
for i in import_path
]
# Handle "magic" Flask extension imports:
# ``flask.ext.foo`` is really ``flask_foo`` or ``flaskext.foo``.
if len(import_path) > 2 and import_parts[:2] == ['flask', 'ext']:
# New style.
ipath = ('flask_' + str(import_parts[2]), ) + import_path[3:]
modules = self._do_import(ipath, sys_path)
if modules:
return modules
else:
# Old style
return self._do_import(('flaskext', ) + import_path[2:],
sys_path)
module_name = '.'.join(import_parts)
try:
return ContextSet(self._evaluator.module_cache.get(module_name))
except KeyError:
pass
if len(import_path) > 1:
# This is a recursive way of importing that works great with
# the module cache.
bases = self._do_import(import_path[:-1], sys_path)
if not bases:
return NO_CONTEXTS
# We can take the first element, because only the os special
# case yields multiple modules, which is not important for
# further imports.
parent_module = list(bases)[0]
# This is a huge exception, we follow a nested import
# ``os.path``, because it's a very important one in Python
# that is being achieved by messing with ``sys.modules`` in
# ``os``.
if import_parts == ['os', 'path']:
return parent_module.py__getattribute__('path')
try:
method = parent_module.py__path__
except AttributeError:
# The module is not a package.
_add_error(self.module_context, import_path[-1])
return NO_CONTEXTS
else:
paths = method()
debug.dbg('search_module %s in paths %s', module_name, paths)
for path in paths:
# At the moment we are only using one path. So this is
# not important to be correct.
if not isinstance(path, list):
path = [path]
code, module_path, is_pkg = self._evaluator.compiled_subprocess.get_module_info(
string=import_parts[-1],
path=path,
full_name=module_name)
if module_path is not None:
break
else:
_add_error(self.module_context, import_path[-1])
return NO_CONTEXTS
else:
debug.dbg('search_module %s in %s', import_parts[-1],
self.file_path)
# Override the sys.path. It works only good that way.
# Injecting the path directly into `find_module` did not work.
code, module_path, is_pkg = self._evaluator.compiled_subprocess.get_module_info(
string=import_parts[-1],
full_name=module_name,
sys_path=sys_path,
)
if module_path is None:
# The module is not a package.
_add_error(self.module_context, import_path[-1])
return NO_CONTEXTS
module = _load_module(
self._evaluator,
module_path,
code,
sys_path,
module_name=module_name,
safe_module_name=True,
)
if module is None:
# The file might raise an ImportError e.g. and therefore not be
# importable.
return NO_CONTEXTS
return ContextSet(module)
def completions(self):
"""
Return :class:`api_classes.Completion` objects. Those objects contain
information about the completions, more than just names.
:return: Completion objects, sorted by name and __ comes last.
:rtype: list of :class:`api_classes.Completion`
"""
debug.speed('completions start')
path = self._module.get_path_until_cursor()
if re.search('^\.|\.\.$', path):
return []
path, dot, like = self._get_completion_parts(path)
completion_line = self._module.get_line(self.pos[0])[:self.pos[1]]
try:
scopes = list(self._prepare_goto(path, True))
except NotFoundError:
scopes = []
scope_generator = evaluate.get_names_of_scope(
self._parser.user_scope, self.pos)
completions = []
for scope, name_list in scope_generator:
for c in name_list:
completions.append((c, scope))
else:
completions = []
debug.dbg('possible scopes', scopes)
for s in scopes:
if s.isinstance(er.Function):
names = s.get_magic_method_names()
else:
if isinstance(s, imports.ImportPath):
if like == 'import':
if not completion_line.endswith('import import'):
continue
a = s.import_stmt.alias
if a and a.start_pos <= self.pos <= a.end_pos:
continue
names = s.get_defined_names(on_import_stmt=True)
else:
names = s.get_defined_names()
for c in names:
completions.append((c, s))
if not dot: # named params have no dots
for call_def in self.call_signatures():
if not call_def.module.is_builtin():
for p in call_def.params:
completions.append((p.get_name(), p))
# Do the completion if there is no path before and no import stmt.
u = self._parser.user_stmt
bs = builtin.Builtin.scope
if isinstance(u, pr.Import):
if (u.relative_count > 0 or u.from_ns) and not re.search(
r'(,|from)\s*$|import\s+$', completion_line):
completions += ((k, bs)
for k in keywords.get_keywords('import'))
if not path and not isinstance(u, pr.Import):
# add keywords
completions += ((k, bs)
for k in keywords.get_keywords(all=True))
needs_dot = not dot and path
comps = []
comp_dct = {}
for c, s in set(completions):
n = c.names[-1]
if settings.case_insensitive_completion \
and n.lower().startswith(like.lower()) \
or n.startswith(like):
if not evaluate.filter_private_variable(
s, self._parser.user_stmt, n):
new = api_classes.Completion(c, needs_dot, len(like), s)
k = (new.name, new.complete) # key
if k in comp_dct and settings.no_completion_duplicates:
comp_dct[k]._same_name_completions.append(new)
else:
comp_dct[k] = new
comps.append(new)
debug.speed('completions end')
return sorted(
comps,
key=lambda x:
(x.name.startswith('__'), x.name.startswith('_'), x.name.lower()))
