def call_of_name(name, cut_own_trailer=False):
"""
Creates a "call" node that consist of all ``trailer`` and ``power``
objects. E.g. if you call it with ``append``::
list([]).append(3) or None
You would get a node with the content ``list([]).append`` back.
This generates a copy of the original ast node.
"""
par = name
if tree.is_node(par.parent, 'trailer'):
par = par.parent
power = par.parent
if tree.is_node(power, 'power') and power.children[0] != name \
and not (power.children[-2] == '**' and
name.start_pos > power.children[-1].start_pos):
par = power
# Now the name must be part of a trailer
index = par.children.index(name.parent)
if index != len(par.children) - 1 or cut_own_trailer:
# Now we have to cut the other trailers away.
par = deep_ast_copy(par)
if not cut_own_trailer:
# Normally we would remove just the stuff after the index, but
# if the option is set remove the index as well. (for goto)
index = index + 1
par.children[index:] = []
return par
def create_index_types(evaluator, index):
"""
Handles slices in subscript nodes.
"""
if index == ':':
# Like array[:]
return set([Slice(evaluator, None, None, None)])
elif tree.is_node(index, 'subscript'): # subscript is a slice operation.
# Like array[:3]
result = []
for el in index.children:
if el == ':':
if not result:
result.append(None)
elif tree.is_node(el, 'sliceop'):
if len(el.children) == 2:
result.append(el.children[1])
else:
result.append(el)
result += [None] * (3 - len(result))
return set([Slice(evaluator, *result)])
# No slices
return evaluator.eval_element(index)
def call_of_name(name, cut_own_trailer=False):
"""
Creates a "call" node that consist of all ``trailer`` and ``power``
objects. E.g. if you call it with ``append``::
list([]).append(3) or None
You would get a node with the content ``list([]).append`` back.
This generates a copy of the original ast node.
"""
par = name
if pr.is_node(par.parent, 'trailer'):
par = par.parent
power = par.parent
if pr.is_node(power, 'power') and power.children[0] != name \
and not (power.children[-2] == '**' and
name.start_pos > power.children[-1].start_pos):
par = power
# Now the name must be part of a trailer
index = par.children.index(name.parent)
if index != len(par.children) - 1 or cut_own_trailer:
# Now we have to cut the other trailers away.
par = deep_ast_copy(par)
if not cut_own_trailer:
# Normally we would remove just the stuff after the index, but
# if the option is set remove the index as well. (for goto)
index = index + 1
par.children[index:] = []
return par
def create_index_types(evaluator, index):
"""
Handles slices in subscript nodes.
"""
if index == ':':
# Like array[:]
return set([Slice(evaluator, None, None, None)])
elif tree.is_node(index, 'subscript'): # subscript is a slice operation.
# Like array[:3]
result = []
for el in index.children:
if el == ':':
if not result:
result.append(None)
elif tree.is_node(el, 'sliceop'):
if len(el.children) == 2:
result.append(el.children[1])
else:
result.append(el)
result += [None] * (3 - len(result))
return set([Slice(evaluator, *result)])
# No slices
return evaluator.eval_element(index)
def get_sys_path_powers(names):
for name in names:
power = name.parent.parent
if tree.is_node(power, "power", "atom_expr"):
c = power.children
if isinstance(c[0], tree.Name) and c[0].value == "sys" and tree.is_node(c[1], "trailer"):
n = c[1].children[1]
if isinstance(n, tree.Name) and n.value == "path":
yield name, power
def get_sys_path_powers(names):
for name in names:
power = name.parent.parent
if pr.is_node(power, 'power'):
c = power.children
if isinstance(c[0], pr.Name) and c[0].value == 'sys' \
and pr.is_node(c[1], 'trailer'):
n = c[1].children[1]
if isinstance(n, pr.Name) and n.value == 'path':
yield name, power
def get_sys_path_powers(names):
for name in names:
power = name.parent.parent
if pr.is_node(power, 'power'):
c = power.children
if isinstance(c[0], pr.Name) and c[0].value == 'sys' \
and pr.is_node(c[1], 'trailer'):
n = c[1].children[1]
if isinstance(n, pr.Name) and n.value == 'path':
yield name, power
def _paths_from_list_modifications(module_path, trailer1, trailer2):
""" extract the path from either "sys.path.append" or "sys.path.insert" """
# Guarantee that both are trailers, the first one a name and the second one
# a function execution with at least one param.
if not (tree.is_node(trailer1, 'trailer') and trailer1.children[0] == '.'
and tree.is_node(trailer2, 'trailer') and trailer2.children[0] == '('
and len(trailer2.children) == 3):
return []
name = trailer1.children[1].value
if name not in ['insert', 'append']:
return []
arg = trailer2.children[1]
if name == 'insert' and len(arg.children) in (3, 4): # Possible trailing comma.
arg = arg.children[2]
return _execute_code(module_path, arg.get_code())
def _eval_atom(self, atom):
"""
Basically to process ``atom`` nodes. The parser sometimes doesn't
generate the node (because it has just one child). In that case an atom
might be a name or a literal as well.
"""
if isinstance(atom, pr.Name):
# This is the first global lookup.
stmt = atom.get_definition()
scope = stmt.get_parent_until(pr.IsScope, include_current=True)
if isinstance(stmt, pr.CompFor):
stmt = stmt.get_parent_until((pr.ClassOrFunc, pr.ExprStmt))
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = atom
return self.find_types(scope, atom, stmt.start_pos, search_global=True)
elif isinstance(atom, pr.Literal):
return [compiled.create(self, atom.eval())]
else:
c = atom.children
# Parentheses without commas are not tuples.
if c[0] == '(' and not len(c) == 2 \
and not(pr.is_node(c[1], 'testlist_comp')
and len(c[1].children) > 1):
return self.eval_element(c[1])
try:
comp_for = c[1].children[1]
except (IndexError, AttributeError):
pass
else:
if isinstance(comp_for, pr.CompFor):
return [iterable.Comprehension.from_atom(self, atom)]
return [iterable.Array(self, atom)]
def _eval_atom(self, atom):
"""
Basically to process ``atom`` nodes. The parser sometimes doesn't
generate the node (because it has just one child). In that case an atom
might be a name or a literal as well.
"""
if isinstance(atom, pr.Name):
# This is the first global lookup.
stmt = atom.get_definition()
scope = stmt.get_parent_until(pr.IsScope, include_current=True)
if isinstance(stmt, pr.CompFor):
stmt = stmt.get_parent_until((pr.ClassOrFunc, pr.ExprStmt))
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = atom
return self.find_types(scope, atom, stmt.start_pos, search_global=True)
elif isinstance(atom, pr.Literal):
return [compiled.create(self, atom.eval())]
else:
c = atom.children
# Parentheses without commas are not tuples.
if c[0] == '(' and not len(c) == 2 \
and not(pr.is_node(c[1], 'testlist_comp')
and len(c[1].children) > 1):
return self.eval_element(c[1])
try:
comp_for = c[1].children[1]
except (IndexError, AttributeError):
pass
else:
if isinstance(comp_for, pr.CompFor):
return [iterable.Comprehension.from_atom(self, atom)]
return [iterable.Array(self, atom)]
def _self_names_dict(self, add_mro=True):
names = {}
# This loop adds the names of the self object, copies them and removes
# the self.
for sub in self.base.subscopes:
if isinstance(sub, pr.Class):
continue
# Get the self name, if there's one.
self_name = self._get_func_self_name(sub)
if self_name is None:
continue
if sub.name.value == '__init__' and not self.is_generated:
# ``__init__`` is special because the params need are injected
# this way. Therefore an execution is necessary.
if not sub.get_decorators():
# __init__ decorators should generally just be ignored,
# because to follow them and their self variables is too
# complicated.
sub = self._get_method_execution(sub)
for name_list in sub.names_dict.values():
for name in name_list:
if name.value == self_name and name.prev_sibling() is None:
trailer = name.next_sibling()
if pr.is_node(trailer, 'trailer') \
and len(trailer.children) == 2 \
and trailer.children[0] == '.':
name = trailer.children[1] # After dot.
if name.is_definition():
arr = names.setdefault(name.value, [])
arr.append(get_instance_el(self._evaluator, self, name))
return names
def calculate_children(evaluator, children):
"""
Calculate a list of children with operators.
"""
iterator = iter(children)
types = evaluator.eval_element(next(iterator))
for operator in iterator:
right = next(iterator)
if tree.is_node(operator, 'comp_op'): # not in / is not
operator = ' '.join(str(c.value) for c in operator.children)
# handle lazy evaluation of and/or here.
if operator in ('and', 'or'):
left_bools = set([left.py__bool__() for left in types])
if left_bools == set([True]):
if operator == 'and':
types = evaluator.eval_element(right)
elif left_bools == set([False]):
if operator != 'and':
types = evaluator.eval_element(right)
# Otherwise continue, because of uncertainty.
else:
types = calculate(evaluator, types, operator,
evaluator.eval_element(right))
debug.dbg('calculate_children types %s', types)
return types
def goto(self, name):
def resolve_implicit_imports(names):
for name in names:
if isinstance(name.parent, helpers.FakeImport):
# Those are implicit imports.
s = imports.ImportWrapper(self, name)
for n in s.follow(is_goto=True):
yield n
yield name
stmt = name.get_definition()
par = name.parent
if par.type == 'argument' and par.children[1] == '=' and par.children[
0] == name:
# Named param goto.
trailer = par.parent
if trailer.type == 'arglist':
trailer = trailer.parent
if trailer.type != 'classdef':
for i, t in enumerate(trailer.parent.children):
if t == trailer:
to_evaluate = trailer.parent.children[:i]
types = self.eval_element(to_evaluate[0])
for trailer in to_evaluate[1:]:
types = self.eval_trailer(types, trailer)
param_names = []
for typ in types:
try:
params = typ.params
except AttributeError:
pass
else:
param_names += [
param.name for param in params
if param.name.value == name.value
]
return param_names
elif isinstance(par, pr.ExprStmt) and name in par.get_defined_names():
# Only take the parent, because if it's more complicated than just
# a name it's something you can "goto" again.
return [name]
elif isinstance(
par, (pr.Param, pr.Function, pr.Class)) and par.name is name:
return [name]
elif isinstance(stmt, pr.Import):
return imports.ImportWrapper(self, name).follow(is_goto=True)
scope = name.get_parent_scope()
if pr.is_node(name.parent, 'trailer'):
call = call_of_name(name, cut_own_trailer=True)
types = self.eval_element(call)
return resolve_implicit_imports(
iterable.unite(
self.find_types(typ, name, is_goto=True) for typ in types))
else:
return self.find_types(scope,
name,
stmt.start_pos,
search_global=True,
is_goto=True)
def as_tuple(self):
for stars, argument in self._split():
if pr.is_node(argument, 'argument'):
argument, default = argument.children[::2]
else:
default = None
yield argument, default, stars
def _eval_element_not_cached(self, element):
debug.dbg('eval_element %s@%s', element, element.start_pos)
types = set()
if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(
element, 'atom'):
types = self._eval_atom(element)
elif isinstance(element, tree.Keyword):
# For False/True/None
if element.value in ('False', 'True', 'None'):
types.add(compiled.builtin_from_name(self, element.value))
# else: print e.g. could be evaluated like this in Python 2.7
elif element.isinstance(tree.Lambda):
types = set([er.LambdaWrapper(self, element)])
elif element.isinstance(er.LambdaWrapper):
types = set([element
]) # TODO this is no real evaluation. id:121 gh:122
elif element.type == 'expr_stmt':
types = self.eval_statement(element)
elif element.type in ('power', 'atom_expr'):
types = self._eval_atom(element.children[0])
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
right = self.eval_element(element.children[2])
types = set(
precedence.calculate(self, types, trailer, right))
break
types = self.eval_trailer(types, trailer)
elif element.type in (
'testlist_star_expr',
'testlist',
):
# The implicit tuple in statements.
types = set([iterable.ImplicitTuple(self, element)])
elif element.type in ('not_test', 'factor'):
types = self.eval_element(element.children[-1])
for operator in element.children[:-1]:
types = set(precedence.factor_calculate(self, types, operator))
elif element.type == 'test':
# `x if foo else y` case.
types = (self.eval_element(element.children[0])
| self.eval_element(element.children[-1]))
elif element.type == 'operator':
# Must be an ellipsis, other operators are not evaluated.
assert element.value == '...'
types = set([compiled.create(self, Ellipsis)])
elif element.type == 'dotted_name':
types = self._eval_atom(element.children[0])
for next_name in element.children[2::2]:
types = set(
chain.from_iterable(
self.find_types(typ, next_name) for typ in types))
types = types
elif element.type == 'eval_input':
types = self._eval_element_not_cached(element.children[0])
elif element.type == 'annassign':
types = self.eval_element(element.children[1])
else:
types = precedence.calculate_children(self, element.children)
debug.dbg('eval_element result %s', types)
return types
def calculate_children(evaluator, children):
"""
Calculate a list of children with operators.
"""
iterator = iter(children)
types = evaluator.eval_element(next(iterator))
for operator in iterator:
right = next(iterator)
if pr.is_node(operator, 'comp_op'): # not in / is not
operator = ' '.join(str(c.value) for c in operator.children)
# handle lazy evaluation of and/or here.
if operator in ('and', 'or'):
left_bools = set([left.py__bool__() for left in types])
if left_bools == set([True]):
if operator == 'and':
types = evaluator.eval_element(right)
elif left_bools == set([False]):
if operator != 'and':
types = evaluator.eval_element(right)
# Otherwise continue, because of uncertainty.
else:
types = calculate(evaluator, types, operator,
evaluator.eval_element(right))
debug.dbg('calculate_children types %s', types)
return types
def _paths_from_list_modifications(module_path, trailer1, trailer2):
""" extract the path from either "sys.path.append" or "sys.path.insert" """
# Guarantee that both are trailers, the first one a name and the second one
# a function execution with at least one param.
if not (tree.is_node(trailer1, 'trailer') and trailer1.children[0] == '.'
and tree.is_node(trailer2, 'trailer') and trailer2.children[0] == '('
and len(trailer2.children) == 3):
return []
name = trailer1.children[1].value
if name not in ['insert', 'append']:
return []
arg = trailer2.children[1]
if name == 'insert' and len(arg.children) in (3, 4): # Possible trailing comma.
arg = arg.children[2]
return _execute_code(module_path, arg.get_code())
def unpack(self, func=None):
named_args = []
for stars, el in self._split():
if stars == 1:
arrays = self._evaluator.eval_element(el)
iterators = [
_iterate_star_args(self._evaluator, a, el, func)
for a in arrays
]
iterators = list(iterators)
for values in list(zip_longest(*iterators)):
yield None, [v for v in values if v is not None]
elif stars == 2:
arrays = self._evaluator.eval_element(el)
dicts = [
_star_star_dict(self._evaluator, a, el, func)
for a in arrays
]
for dct in dicts:
for key, values in dct.items():
yield key, values
else:
if pr.is_node(el, 'argument'):
named_args.append(
(el.children[0].value, (el.children[2], )))
elif isinstance(el, (list, tuple)):
yield None, el
else:
yield None, (el, )
# Reordering var_args is necessary, because star args sometimes appear
# after named argument, but in the actual order it's prepended.
for key_arg in named_args:
yield key_arg
def as_tuple(self):
for stars, argument in self._split():
if tree.is_node(argument, 'argument'):
argument, default = argument.children[::2]
else:
default = None
yield argument, default, stars
def _self_names_dict(self, add_mro=True):
names = {}
# This loop adds the names of the self object, copies them and removes
# the self.
for sub in self.base.subscopes:
if isinstance(sub, pr.Class):
continue
# Get the self name, if there's one.
self_name = self._get_func_self_name(sub)
if self_name is None:
continue
if sub.name.value == '__init__' and not self.is_generated:
# ``__init__`` is special because the params need are injected
# this way. Therefore an execution is necessary.
if not sub.get_decorators():
# __init__ decorators should generally just be ignored,
# because to follow them and their self variables is too
# complicated.
sub = self._get_method_execution(sub)
for name_list in sub.names_dict.values():
for name in name_list:
if name.value == self_name and name.prev_sibling() is None:
trailer = name.next_sibling()
if pr.is_node(trailer, 'trailer') \
and len(trailer.children) == 2 \
and trailer.children[0] == '.':
name = trailer.children[1] # After dot.
if name.is_definition():
arr = names.setdefault(name.value, [])
arr.append(
get_instance_el(self._evaluator, self,
name))
return names
def _eval_atom(self, atom):
"""
Basically to process ``atom`` nodes. The parser sometimes doesn't
generate the node (because it has just one child). In that case an atom
might be a name or a literal as well.
"""
if isinstance(atom, tree.Name):
# This is the first global lookup.
stmt = atom.get_definition()
scope = stmt.get_parent_until(tree.IsScope, include_current=True)
if isinstance(scope, (tree.Function, er.FunctionExecution)):
# Adjust scope: If the name is not in the suite, it's a param
# default or annotation and will be resolved as part of the
# parent scope.
colon = scope.children.index(':')
if atom.start_pos < scope.children[colon + 1].start_pos:
scope = scope.get_parent_scope()
if isinstance(stmt, tree.CompFor):
stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt))
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = atom
return self.find_types(scope,
atom,
stmt.start_pos,
search_global=True)
elif isinstance(atom, tree.Literal):
return set([compiled.create(self, atom.eval())])
else:
c = atom.children
if c[0].type == 'string':
# Will be one string.
types = self._eval_atom(c[0])
for string in c[1:]:
right = self._eval_atom(string)
types = precedence.calculate(self, types, '+', right)
return types
# Parentheses without commas are not tuples.
elif c[0] == '(' and not len(c) == 2 \
and not(tree.is_node(c[1], 'testlist_comp')
and len(c[1].children) > 1):
return self.eval_element(c[1])
try:
comp_for = c[1].children[1]
except (IndexError, AttributeError):
pass
else:
if comp_for == ':':
# Dict comprehensions have a colon at the 3rd index.
try:
comp_for = c[1].children[3]
except IndexError:
pass
if comp_for.type == 'comp_for':
return set([iterable.Comprehension.from_atom(self, atom)])
return set([iterable.Array(self, atom)])
def create_indexes_or_slices(evaluator, index):
if tree.is_node(index, 'subscript'): # subscript is a slice operation.
start, stop, step = None, None, None
result = []
for el in index.children:
if el == ':':
if not result:
result.append(None)
elif tree.is_node(el, 'sliceop'):
if len(el.children) == 2:
result.append(el.children[1])
else:
result.append(el)
result += [None] * (3 - len(result))
return (Slice(evaluator, *result),)
return evaluator.eval_element(index)
def create_indexes_or_slices(evaluator, index):
if pr.is_node(index, 'subscript'): # subscript is a slice operation.
start, stop, step = None, None, None
result = []
for el in index.children:
if el == ':':
if not result:
result.append(None)
elif pr.is_node(el, 'sliceop'):
if len(el.children) == 2:
result.append(el.children[1])
else:
result.append(el)
result += [None] * (3 - len(result))
return (Slice(evaluator, *result), )
return evaluator.eval_element(index)
def eval_element(self, element):
if isinstance(element, iterable.AlreadyEvaluated):
return list(element)
elif isinstance(element, iterable.MergedNodes):
return iterable.unite(self.eval_element(e) for e in element)
debug.dbg('eval_element %s@%s', element, element.start_pos)
if isinstance(element,
(pr.Name, pr.Literal)) or pr.is_node(element, 'atom'):
return self._eval_atom(element)
elif isinstance(element, pr.Keyword):
# For False/True/None
if element.value in ('False', 'True', 'None'):
return [compiled.builtin.get_by_name(element.value)]
else:
return []
elif element.isinstance(pr.Lambda):
return [er.LambdaWrapper(self, element)]
elif element.isinstance(er.LambdaWrapper):
return [element] # TODO this is no real evaluation.
elif element.type == 'expr_stmt':
return self.eval_statement(element)
elif element.type == 'power':
types = self._eval_atom(element.children[0])
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
raise NotImplementedError
types = self.eval_trailer(types, trailer)
return types
elif element.type in (
'testlist_star_expr',
'testlist',
):
# The implicit tuple in statements.
return [iterable.ImplicitTuple(self, element)]
elif element.type in ('not_test', 'factor'):
types = self.eval_element(element.children[-1])
for operator in element.children[:-1]:
types = list(precedence.factor_calculate(
self, types, operator))
return types
elif element.type == 'test':
# `x if foo else y` case.
return (self.eval_element(element.children[0]) +
self.eval_element(element.children[-1]))
elif element.type == 'operator':
# Must be an ellipsis, other operators are not evaluated.
return [] # Ignore for now.
elif element.type == 'dotted_name':
types = self._eval_atom(element.children[0])
for next_name in element.children[2::2]:
types = list(
chain.from_iterable(
self.find_types(typ, next_name) for typ in types))
return types
else:
return precedence.calculate_children(self, element.children)
def get_posibilities(evaluator, module, func_name):
try:
names = module.used_names[func_name]
except KeyError:
return []
for name in names:
stmt = name.get_definition()
if not isinstance(stmt,
(pr.ExprStmt, pr.CompFor, pr.ReturnStmt)):
continue
parent = name.parent
if pr.is_node(parent, 'trailer'):
parent = parent.parent
trailer = None
if pr.is_node(parent, 'power'):
for t in parent.children[1:]:
if t == '**':
break
if t.start_pos > name.start_pos and t.children[
0] == '(':
trailer = t
break
if trailer is not None:
types = evaluator.goto_definition(name)
# We have to remove decorators, because they are not the
# "original" functions, this way we can easily compare.
# At the same time we also have to remove InstanceElements.
undec = []
for escope in types:
if escope.isinstance(er.Function, er.Instance) \
and escope.decorates is not None:
undec.append(escope.decorates)
elif isinstance(escope, er.InstanceElement):
undec.append(escope.var)
else:
undec.append(escope)
if er.wrap(evaluator, compare) in undec:
# Only if we have the correct function we execute
# it, otherwise just ignore it.
evaluator.eval_trailer(types, trailer)
return listener.param_possibilities
def _eval_atom(self, atom):
"""
Basically to process ``atom`` nodes. The parser sometimes doesn't
generate the node (because it has just one child). In that case an atom
might be a name or a literal as well.
"""
if isinstance(atom, tree.Name):
# This is the first global lookup.
stmt = atom.get_definition()
scope = stmt.get_parent_until(tree.IsScope, include_current=True)
if isinstance(scope, (tree.Function, er.FunctionExecution)):
# Adjust scope: If the name is not in the suite, it's a param
# default or annotation and will be resolved as part of the
# parent scope.
colon = scope.children.index(':')
if atom.start_pos < scope.children[colon + 1].start_pos:
scope = scope.get_parent_scope()
if isinstance(stmt, tree.CompFor):
stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt))
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = atom
return self.find_types(scope, atom, stmt.start_pos, search_global=True)
elif isinstance(atom, tree.Literal):
return set([compiled.create(self, atom.eval())])
else:
c = atom.children
if c[0].type == 'string':
# Will be one string.
types = self._eval_atom(c[0])
for string in c[1:]:
right = self._eval_atom(string)
types = precedence.calculate(self, types, '+', right)
return types
# Parentheses without commas are not tuples.
elif c[0] == '(' and not len(c) == 2 \
and not(tree.is_node(c[1], 'testlist_comp')
and len(c[1].children) > 1):
return self.eval_element(c[1])
try:
comp_for = c[1].children[1]
except (IndexError, AttributeError):
pass
else:
if comp_for == ':':
# Dict comprehensions have a colon at the 3rd index.
try:
comp_for = c[1].children[3]
except IndexError:
pass
if comp_for.type == 'comp_for':
return set([iterable.Comprehension.from_atom(self, atom)])
return set([iterable.Array(self, atom)])
def _eval_element_not_cached(self, element):
debug.dbg('eval_element %s@%s', element, element.start_pos)
types = set()
if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'):
types = self._eval_atom(element)
elif isinstance(element, tree.Keyword):
# For False/True/None
if element.value in ('False', 'True', 'None'):
types.add(compiled.builtin_from_name(self, element.value))
# else: print e.g. could be evaluated like this in Python 2.7
elif element.isinstance(tree.Lambda):
types = set([er.LambdaWrapper(self, element)])
elif element.isinstance(er.LambdaWrapper):
types = set([element]) # TODO this is no real evaluation.
elif element.type == 'expr_stmt':
types = self.eval_statement(element)
elif element.type in ('power', 'atom_expr'):
types = self._eval_atom(element.children[0])
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
right = self.eval_element(element.children[2])
types = set(precedence.calculate(self, types, trailer, right))
break
types = self.eval_trailer(types, trailer)
elif element.type in ('testlist_star_expr', 'testlist',):
# The implicit tuple in statements.
types = set([iterable.ImplicitTuple(self, element)])
elif element.type in ('not_test', 'factor'):
types = self.eval_element(element.children[-1])
for operator in element.children[:-1]:
types = set(precedence.factor_calculate(self, types, operator))
elif element.type == 'test':
# `x if foo else y` case.
types = (self.eval_element(element.children[0]) |
self.eval_element(element.children[-1]))
elif element.type == 'operator':
# Must be an ellipsis, other operators are not evaluated.
assert element.value == '...'
types = set([compiled.create(self, Ellipsis)])
elif element.type == 'dotted_name':
types = self._eval_atom(element.children[0])
for next_name in element.children[2::2]:
types = set(chain.from_iterable(self.find_types(typ, next_name)
for typ in types))
types = types
elif element.type == 'eval_input':
types = self._eval_element_not_cached(element.children[0])
elif element.type == 'annassign':
types = self.eval_element(element.children[1])
else:
types = precedence.calculate_children(self, element.children)
debug.dbg('eval_element result %s', types)
return types
def get_posibilities(evaluator, module, func_name):
try:
names = module.used_names[func_name]
except KeyError:
return []
for name in names:
parent = name.parent
if tree.is_node(parent, 'trailer'):
parent = parent.parent
trailer = None
if tree.is_node(parent, 'power'):
for t in parent.children[1:]:
if t == '**':
break
if t.start_pos > name.start_pos and t.children[0] == '(':
trailer = t
break
if trailer is not None:
types = evaluator.goto_definition(name)
# We have to remove decorators, because they are not the
# "original" functions, this way we can easily compare.
# At the same time we also have to remove InstanceElements.
undec = []
for escope in types:
if escope.isinstance(er.Function, er.Instance) \
and escope.decorates is not None:
undec.append(escope.decorates)
elif isinstance(escope, er.InstanceElement):
undec.append(escope.var)
else:
undec.append(escope)
if evaluator.wrap(compare) in undec:
# Only if we have the correct function we execute
# it, otherwise just ignore it.
evaluator.eval_trailer(types, trailer)
return listener.param_possibilities
def eval_element(self, element):
if isinstance(element, iterable.AlreadyEvaluated):
return list(element)
elif isinstance(element, iterable.MergedNodes):
return iterable.unite(self.eval_element(e) for e in element)
debug.dbg('eval_element %s@%s', element, element.start_pos)
if isinstance(element, (pr.Name, pr.Literal)) or pr.is_node(element, 'atom'):
return self._eval_atom(element)
elif isinstance(element, pr.Keyword):
# For False/True/None
if element.value in ('False', 'True', 'None'):
return [compiled.builtin.get_by_name(element.value)]
else:
return []
elif element.isinstance(pr.Lambda):
return [er.LambdaWrapper(self, element)]
elif element.isinstance(er.LambdaWrapper):
return [element] # TODO this is no real evaluation.
elif element.type == 'expr_stmt':
return self.eval_statement(element)
elif element.type == 'power':
types = self._eval_atom(element.children[0])
for trailer in element.children[1:]:
if trailer == '**': # has a power operation.
raise NotImplementedError
types = self.eval_trailer(types, trailer)
return types
elif element.type in ('testlist_star_expr', 'testlist',):
# The implicit tuple in statements.
return [iterable.ImplicitTuple(self, element)]
elif element.type in ('not_test', 'factor'):
types = self.eval_element(element.children[-1])
for operator in element.children[:-1]:
types = list(precedence.factor_calculate(self, types, operator))
return types
elif element.type == 'test':
# `x if foo else y` case.
return (self.eval_element(element.children[0]) +
self.eval_element(element.children[-1]))
elif element.type == 'operator':
# Must be an ellipsis, other operators are not evaluated.
return [] # Ignore for now.
elif element.type == 'dotted_name':
types = self._eval_atom(element.children[0])
for next_name in element.children[2::2]:
types = list(chain.from_iterable(self.find_types(typ, next_name)
for typ in types))
return types
else:
return precedence.calculate_children(self, element.children)
def _items(self):
c = self.atom.children
array_node = c[1]
if array_node in (']', '}', ')'):
return [] # Direct closing bracket, doesn't contain items.
if tree.is_node(array_node, 'testlist_comp'):
return array_node.children[::2]
elif tree.is_node(array_node, 'dictorsetmaker'):
kv = []
iterator = iter(array_node.children)
for key in iterator:
op = next(iterator, None)
if op is None or op == ',':
kv.append(key) # A set.
else:
assert op == ':' # A dict.
kv.append((key, next(iterator)))
next(iterator, None) # Possible comma.
return kv
else:
return [array_node]
def _items(self):
c = self.atom.children
array_node = c[1]
if array_node in (']', '}', ')'):
return [] # Direct closing bracket, doesn't contain items.
if tree.is_node(array_node, 'testlist_comp'):
return array_node.children[::2]
elif tree.is_node(array_node, 'dictorsetmaker'):
kv = []
iterator = iter(array_node.children)
for key in iterator:
op = next(iterator, None)
if op is None or op == ',':
kv.append(key) # A set.
else:
assert op == ':' # A dict.
kv.append((key, next(iterator)))
next(iterator, None) # Possible comma.
return kv
else:
return [array_node]
def _items(self):
c = self.atom.children
array_node = c[1]
if array_node in (']', '}', ')'):
return [] # Direct closing bracket, doesn't contain items.
if pr.is_node(array_node, 'testlist_comp'):
return array_node.children[::2]
elif pr.is_node(array_node, 'dictorsetmaker'):
kv = []
iterator = iter(array_node.children)
for key in iterator:
op = next(iterator, None)
if op is None or op == ',':
kv.append(key) # A set.
elif op == ':': # A dict.
kv.append((key, [next(iterator)]))
next(iterator, None) # Possible comma.
else:
raise NotImplementedError('dict/set comprehensions')
return kv
else:
return [array_node]
def _items(self):
c = self.atom.children
array_node = c[1]
if array_node in (']', '}', ')'):
return [] # Direct closing bracket, doesn't contain items.
if pr.is_node(array_node, 'testlist_comp'):
return array_node.children[::2]
elif pr.is_node(array_node, 'dictorsetmaker'):
kv = []
iterator = iter(array_node.children)
for key in iterator:
op = next(iterator, None)
if op is None or op == ',':
kv.append(key) # A set.
elif op == ':': # A dict.
kv.append((key, [next(iterator)]))
next(iterator, None) # Possible comma.
else:
raise NotImplementedError('dict/set comprehensions')
return kv
else:
return [array_node]
def _rewrite_last_newline(self):
"""
The ENDMARKER can contain a newline in the prefix. However this prefix
really belongs to the function - respectively to the next function or
parser node. If we don't rewrite that newline, we end up with a newline
in the wrong position, i.d. at the end of the file instead of in the
middle.
"""
c = self._content_scope.children
if tree.is_node(c[-1], 'suite'): # In a simple_stmt there's no DEDENT.
end_marker = self.parser.module.children[-1]
# Set the DEDENT prefix instead of the ENDMARKER.
c[-1].children[-1].prefix = end_marker.prefix
end_marker.prefix = ''
def _rewrite_last_newline(self):
"""
The ENDMARKER can contain a newline in the prefix. However this prefix
really belongs to the function - respectively to the next function or
parser node. If we don't rewrite that newline, we end up with a newline
in the wrong position, i.d. at the end of the file instead of in the
middle.
"""
c = self._content_scope.children
if pr.is_node(c[-1], 'suite'): # In a simple_stmt there's no DEDENT.
end_marker = self.parser.module.children[-1]
# Set the DEDENT prefix instead of the ENDMARKER.
c[-1].children[-1].prefix = end_marker.prefix
end_marker.prefix = ''
def _split(self):
if isinstance(self.argument_node, (tuple, list)):
for el in self.argument_node:
yield 0, el
else:
if not (tree.is_node(self.argument_node, 'arglist') or (
# in python 3.5 **arg is an argument, not arglist
(tree.is_node(self.argument_node, 'argument')
and self.argument_node.children[0] in ('*', '**')))):
yield 0, self.argument_node
return
iterator = iter(self.argument_node.children)
for child in iterator:
if child == ',':
continue
elif child in ('*', '**'):
yield len(child.value), next(iterator)
elif tree.is_node(child, 'argument') and \
child.children[0] in ('*', '**'):
assert len(child.children) == 2
yield len(child.children[0].value), child.children[1]
else:
yield 0, child
def _split(self):
if isinstance(self.argument_node, (tuple, list)):
for el in self.argument_node:
yield 0, el
else:
if not (tree.is_node(self.argument_node, 'arglist') or (
# in python 3.5 **arg is an argument, not arglist
(tree.is_node(self.argument_node, 'argument') and
self.argument_node.children[0] in ('*', '**')))):
yield 0, self.argument_node
return
iterator = iter(self.argument_node.children)
for child in iterator:
if child == ',':
continue
elif child in ('*', '**'):
yield len(child.value), next(iterator)
elif tree.is_node(child, 'argument') and \
child.children[0] in ('*', '**'):
assert len(child.children) == 2
yield len(child.children[0].value), child.children[1]
else:
yield 0, child
def _split(self):
if isinstance(self.argument_node, (tuple, list)):
for el in self.argument_node:
yield 0, el
else:
if not tree.is_node(self.argument_node, 'arglist'):
yield 0, self.argument_node
return
iterator = iter(self.argument_node.children)
for child in iterator:
if child == ',':
continue
elif child in ('*', '**'):
yield len(child.value), next(iterator)
else:
yield 0, child
def _split(self):
if isinstance(self.argument_node, (tuple, list)):
for el in self.argument_node:
yield 0, el
else:
if not pr.is_node(self.argument_node, 'arglist'):
yield 0, self.argument_node
return
iterator = iter(self.argument_node.children)
for child in iterator:
if child == ',':
continue
elif child in ('*', '**'):
yield len(child.value), next(iterator)
else:
yield 0, child
def eval_node(self):
"""
The first part `x + 1` of the list comprehension:
[x + 1 for x in foo]
"""
comprehension = self._atom.children[1]
# For nested comprehensions we need to search the last one.
last = comprehension.children[-1]
last_comp = comprehension.children[1]
while True:
if isinstance(last, tree.CompFor):
last_comp = last
elif not tree.is_node(last, 'comp_if'):
break
last = last.children[-1]
return helpers.deep_ast_copy(comprehension.children[0], parent=last_comp)
def eval_node(self):
"""
The first part `x + 1` of the list comprehension:
[x + 1 for x in foo]
"""
comprehension = self._atom.children[1]
# For nested comprehensions we need to search the last one.
last = comprehension.children[-1]
last_comp = comprehension.children[1]
while True:
if isinstance(last, pr.CompFor):
last_comp = last
elif not pr.is_node(last, 'comp_if'):
break
last = last.children[-1]
return helpers.deep_ast_copy(comprehension.children[0],
parent=last_comp)
def _paths_from_assignment(evaluator, expr_stmt):
"""
Extracts the assigned strings from an assignment that looks as follows::
>>> sys.path[0:0] = ['module/path', 'another/module/path']
This function is in general pretty tolerant (and therefore 'buggy').
However, it's not a big issue usually to add more paths to Jedi's sys_path,
because it will only affect Jedi in very random situations and by adding
more paths than necessary, it usually benefits the general user.
"""
for assignee, operator in zip(expr_stmt.children[::2],
expr_stmt.children[1::2]):
try:
assert operator in ['=', '+=']
assert tree.is_node(assignee, 'power', 'atom_expr') and \
len(assignee.children) > 1
c = assignee.children
assert c[0].type == 'name' and c[0].value == 'sys'
trailer = c[1]
assert trailer.children[0] == '.' and trailer.children[
1].value == 'path'
# TODO Essentially we're not checking details on sys.path
# manipulation. Both assigment of the sys.path and changing/adding
# parts of the sys.path are the same: They get added to the current
# sys.path.
"""
execution = c[2]
assert execution.children[0] == '['
subscript = execution.children[1]
assert subscript.type == 'subscript'
assert ':' in subscript.children
"""
except AssertionError:
continue
from jedi.evaluate.iterable import py__iter__
from jedi.evaluate.precedence import is_string
types = evaluator.eval_element(expr_stmt)
for types in py__iter__(evaluator, types, expr_stmt):
for typ in types:
if is_string(typ):
yield typ.obj
def _paths_from_assignment(evaluator, expr_stmt):
"""
Extracts the assigned strings from an assignment that looks as follows::
>>> sys.path[0:0] = ['module/path', 'another/module/path']
This function is in general pretty tolerant (and therefore 'buggy').
However, it's not a big issue usually to add more paths to Jedi's sys_path,
because it will only affect Jedi in very random situations and by adding
more paths than necessary, it usually benefits the general user.
"""
for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]):
try:
assert operator in ['=', '+=']
assert tree.is_node(assignee, 'power', 'atom_expr') and \
len(assignee.children) > 1
c = assignee.children
assert c[0].type == 'name' and c[0].value == 'sys'
trailer = c[1]
assert trailer.children[0] == '.' and trailer.children[1].value == 'path'
# TODO Essentially we're not checking details on sys.path
# manipulation. Both assigment of the sys.path and changing/adding
# parts of the sys.path are the same: They get added to the current
# sys.path.
"""
execution = c[2]
assert execution.children[0] == '['
subscript = execution.children[1]
assert subscript.type == 'subscript'
assert ':' in subscript.children
"""
except AssertionError:
continue
from jedi.evaluate.iterable import py__iter__
from jedi.evaluate.precedence import is_string
types = evaluator.eval_element(expr_stmt)
for types in py__iter__(evaluator, types, expr_stmt):
for typ in types:
if is_string(typ):
yield typ.obj
def get_types_for_typing_module(evaluator, typ, node):
from jedi.evaluate.iterable import FakeSequence
if not typ.base.get_parent_until().name.value == "typing":
return None
# we assume that any class using [] in a module called
# "typing" with a name for which we have a replacement
# should be replaced by that class. This is not 100%
# airtight but I don't have a better idea to check that it's
# actually the PEP-0484 typing module and not some other
if tree.is_node(node, "subscriptlist"):
nodes = node.children[::2] # skip the commas
else:
nodes = [node]
del node
nodes = [_fix_forward_reference(evaluator, node) for node in nodes]
# hacked in Union and Optional, since it's hard to do nicely in parsed code
if typ.name.value == "Union":
return unite(evaluator.eval_element(node) for node in nodes)
if typ.name.value == "Optional":
return evaluator.eval_element(nodes[0])
typing = _get_typing_replacement_module()
factories = evaluator.find_types(typing, "factory")
assert len(factories) == 1
factory = list(factories)[0]
assert factory
function_body_nodes = factory.children[4].children
valid_classnames = set(child.name.value
for child in function_body_nodes
if isinstance(child, tree.Class))
if typ.name.value not in valid_classnames:
return None
compiled_classname = compiled.create(evaluator, typ.name.value)
args = FakeSequence(evaluator, nodes, "tuple")
result = evaluator.execute_evaluated(factory, compiled_classname, args)
return result
def unpack(self, func=None):
named_args = []
for stars, el in self._split():
if stars == 1:
arrays = self._evaluator.eval_element(el)
iterators = [
_iterate_star_args(self._evaluator, a, el, func)
for a in arrays
]
iterators = list(iterators)
for values in list(zip_longest(*iterators)):
yield None, [v for v in values if v is not None]
elif stars == 2:
arrays = self._evaluator.eval_element(el)
dicts = [
_star_star_dict(self._evaluator, a, el, func)
for a in arrays
]
for dct in dicts:
for key, values in dct.items():
yield key, values
else:
if pr.is_node(el, 'argument'):
c = el.children
if len(c) == 3: # Keyword argument.
named_args.append((c[0].value, (c[2], )))
else: # Generator comprehension.
# Include the brackets with the parent.
comp = iterable.GeneratorComprehension(
self._evaluator, self.argument_node.parent)
yield None, (iterable.AlreadyEvaluated([comp]), )
elif isinstance(el, (list, tuple)):
yield None, el
else:
yield None, (el, )
# Reordering var_args is necessary, because star args sometimes appear
# after named argument, but in the actual order it's prepended.
for key_arg in named_args:
yield key_arg
def unpack(self, func=None):
named_args = []
for stars, el in self._split():
if stars == 1:
arrays = self._evaluator.eval_element(el)
iterators = [_iterate_star_args(self._evaluator, a, el, func)
for a in arrays]
iterators = list(iterators)
for values in list(zip_longest(*iterators)):
yield None, [v for v in values if v is not None]
elif stars == 2:
arrays = self._evaluator.eval_element(el)
dicts = [_star_star_dict(self._evaluator, a, el, func)
for a in arrays]
for dct in dicts:
for key, values in dct.items():
yield key, values
else:
if tree.is_node(el, 'argument'):
c = el.children
if len(c) == 3: # Keyword argument.
named_args.append((c[0].value, (c[2],)))
else: # Generator comprehension.
# Include the brackets with the parent.
comp = iterable.GeneratorComprehension(
self._evaluator, self.argument_node.parent)
yield None, (iterable.AlreadyEvaluated([comp]),)
elif isinstance(el, (list, tuple)):
yield None, el
else:
yield None, (el,)
# Reordering var_args is necessary, because star args sometimes appear
# after named argument, but in the actual order it's prepended.
for key_arg in named_args:
yield key_arg
def get_calling_var_args(self):
if tree.is_node(self.argument_node, 'arglist', 'argument') \
or self.argument_node == () and self.trailer is not None:
return _get_calling_var_args(self._evaluator, self)
else:
return None
def get_calling_var_args(self):
if pr.is_node(self.argument_node, 'arglist', 'argument') \
or self.argument_node == () and self.trailer is not None:
return _get_calling_var_args(self._evaluator, self)
else:
return None
def goto(self, name):
def resolve_implicit_imports(names):
for name in names:
if isinstance(name.parent, helpers.FakeImport):
# Those are implicit imports.
s = imports.ImportWrapper(self, name)
for n in s.follow(is_goto=True):
yield n
else:
yield name
stmt = name.get_definition()
par = name.parent
if par.type == 'argument' and par.children[1] == '=' and par.children[
0] == name:
# Named param goto.
trailer = par.parent
if trailer.type == 'arglist':
trailer = trailer.parent
if trailer.type != 'classdef':
if trailer.type == 'decorator':
types = self.eval_element(trailer.children[1])
else:
i = trailer.parent.children.index(trailer)
to_evaluate = trailer.parent.children[:i]
types = self.eval_element(to_evaluate[0])
for trailer in to_evaluate[1:]:
types = self.eval_trailer(types, trailer)
param_names = []
for typ in types:
try:
params = typ.params
except AttributeError:
pass
else:
param_names += [
param.name for param in params
if param.name.value == name.value
]
return param_names
elif isinstance(par,
tree.ExprStmt) and name in par.get_defined_names():
# Only take the parent, because if it's more complicated than just
# a name it's something you can "goto" again.
return [name]
elif isinstance(
par,
(tree.Param, tree.Function, tree.Class)) and par.name is name:
return [name]
elif isinstance(stmt, tree.Import):
modules = imports.ImportWrapper(self, name).follow(is_goto=True)
return list(resolve_implicit_imports(modules))
elif par.type == 'dotted_name': # Is a decorator.
index = par.children.index(name)
if index > 0:
new_dotted = helpers.deep_ast_copy(par)
new_dotted.children[index - 1:] = []
types = self.eval_element(new_dotted)
return resolve_implicit_imports(
iterable.unite(
self.find_types(typ, name, is_goto=True)
for typ in types))
scope = name.get_parent_scope()
if tree.is_node(par, 'trailer') and par.children[0] == '.':
call = helpers.call_of_leaf(name, cut_own_trailer=True)
types = self.eval_element(call)
return resolve_implicit_imports(
iterable.unite(
self.find_types(typ, name, is_goto=True) for typ in types))
else:
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = name
return self.find_types(scope,
name,
stmt.start_pos,
search_global=True,
is_goto=True)
def goto(self, name):
def resolve_implicit_imports(names):
for name in names:
if isinstance(name.parent, helpers.FakeImport):
# Those are implicit imports.
s = imports.ImportWrapper(self, name)
for n in s.follow(is_goto=True):
yield n
else:
yield name
stmt = name.get_definition()
par = name.parent
if par.type == 'argument' and par.children[1] == '=' and par.children[0] == name:
# Named param goto.
trailer = par.parent
if trailer.type == 'arglist':
trailer = trailer.parent
if trailer.type != 'classdef':
if trailer.type == 'decorator':
types = self.eval_element(trailer.children[1])
else:
i = trailer.parent.children.index(trailer)
to_evaluate = trailer.parent.children[:i]
types = self.eval_element(to_evaluate[0])
for trailer in to_evaluate[1:]:
types = self.eval_trailer(types, trailer)
param_names = []
for typ in types:
try:
params = typ.params
except AttributeError:
pass
else:
param_names += [param.name for param in params
if param.name.value == name.value]
return param_names
elif isinstance(par, tree.ExprStmt) and name in par.get_defined_names():
# Only take the parent, because if it's more complicated than just
# a name it's something you can "goto" again.
return [name]
elif isinstance(par, (tree.Param, tree.Function, tree.Class)) and par.name is name:
return [name]
elif isinstance(stmt, tree.Import):
modules = imports.ImportWrapper(self, name).follow(is_goto=True)
return list(resolve_implicit_imports(modules))
elif par.type == 'dotted_name': # Is a decorator.
index = par.children.index(name)
if index > 0:
new_dotted = helpers.deep_ast_copy(par)
new_dotted.children[index - 1:] = []
types = self.eval_element(new_dotted)
return resolve_implicit_imports(iterable.unite(
self.find_types(typ, name, is_goto=True) for typ in types
))
scope = name.get_parent_scope()
if tree.is_node(par, 'trailer') and par.children[0] == '.':
call = helpers.call_of_leaf(name, cut_own_trailer=True)
types = self.eval_element(call)
return resolve_implicit_imports(iterable.unite(
self.find_types(typ, name, is_goto=True) for typ in types
))
else:
if stmt.type != 'expr_stmt':
# We only need to adjust the start_pos for statements, because
# there the name cannot be used.
stmt = name
return self.find_types(scope, name, stmt.start_pos,
search_global=True, is_goto=True)
