def _eval_statement_element(self, element):
if pr.Array.is_type(element, pr.Array.NOARRAY):
r = list(itertools.chain.from_iterable(self.eval_statement(s)
for s in element))
call_path = element.generate_call_path()
next(call_path, None) # the first one has been used already
return self.follow_path(call_path, r, element.parent)
elif isinstance(element, pr.ListComprehension):
loop = _evaluate_list_comprehension(element)
# Caveat: parents are being changed, but this doesn't matter,
# because nothing else uses it.
element.stmt.parent = loop
return self.eval_statement(element.stmt)
elif isinstance(element, pr.Lambda):
return [er.Function(self, element)]
# With things like params, these can also be functions...
elif isinstance(element, pr.Base) and element.isinstance(
er.Function, er.Class, er.Instance, iterable.ArrayInstance):
return [element]
# The string tokens are just operations (+, -, etc.)
elif isinstance(element, compiled.CompiledObject):
return [element]
elif not isinstance(element, Token):
return self.eval_call(element)
else:
return []
def _parse_dot_name(self, pre_used_token=None):
"""
The dot name parser parses a name, variable or function and returns
their names.
:return: tuple of Name, next_token
"""
def append(el):
names.append(el)
self.module.temp_used_names.append(el[0])
names = []
tok = next(self._gen) if pre_used_token is None else pre_used_token
if tok.type != tokenize.NAME and tok.string != '*':
return None, tok
first_pos = tok.start_pos
append((tok.string, first_pos))
while True:
end_pos = tok.end_pos
tok = next(self._gen)
if tok.string != '.':
break
tok = next(self._gen)
if tok.type != tokenize.NAME:
break
append((tok.string, tok.start_pos))
n = pr.Name(self.module, names, first_pos, end_pos) if names else None
return n, tok
def eval_statement_element(self, element):
if pr.Array.is_type(element, pr.Array.NOARRAY):
try:
lst_cmp = element[0].expression_list()[0]
if not isinstance(lst_cmp, pr.ListComprehension):
raise IndexError
except IndexError:
r = list(itertools.chain.from_iterable(self.eval_statement(s)
for s in element))
else:
r = [iterable.GeneratorComprehension(self, lst_cmp)]
call_path = element.generate_call_path()
next(call_path, None) # the first one has been used already
return self.follow_path(call_path, r, element.parent)
elif isinstance(element, pr.ListComprehension):
return self.eval_statement(element.stmt)
elif isinstance(element, pr.Lambda):
return [er.Function(self, element)]
# With things like params, these can also be functions...
elif isinstance(element, pr.Base) and element.isinstance(
er.Function, er.Class, er.Instance, iterable.ArrayInstance):
return [element]
# The string tokens are just operations (+, -, etc.)
elif isinstance(element, compiled.CompiledObject):
return [element]
elif isinstance(element, Token):
return []
else:
return self.eval_call(element)
def _parse_dotted_name(self, pre_used_token=None):
"""
The dot name parser parses a name, variable or function and returns
their names.
Just used for parsing imports.
:return: tuple of Name, next_token
"""
def append(tok):
names.append(pr.Name(self.module, tok.string, None, tok.start_pos))
self.module.temp_used_names.append(tok.string)
names = []
tok = next(self._gen) if pre_used_token is None else pre_used_token
if tok.type != tokenize.NAME and tok.string != '*':
return [], tok
append(tok)
while True:
tok = next(self._gen)
if tok.string != '.':
break
tok = next(self._gen)
if tok.type != tokenize.NAME:
break
append(tok)
return names, tok
def _parse_class(self):
"""
The parser for a text class. Process the tokens, which follow a
class definition.
:return: Return a Scope representation of the tokens.
:rtype: Class
"""
first_pos = self._gen.current.start_pos
cname = next(self._gen)
if cname.type != tokenize.NAME:
debug.warning("class: syntax err, token is not a name@%s (%s: %s)",
cname.start_pos[0], tokenize.tok_name[cname.type], cname.string)
return None
cname = pr.Name(self.module, [(cname.string, cname.start_pos)],
cname.start_pos, cname.end_pos)
super = []
_next = next(self._gen)
if _next.string == '(':
super = self._parse_parentheses()
_next = next(self._gen)
if _next.string != ':':
debug.warning("class syntax: %s@%s", cname, _next.start_pos[0])
return None
return pr.Class(self.module, cname, super, first_pos)
def eval_statement_element(self, element):
if pr.Array.is_type(element, pr.Array.NOARRAY):
try:
lst_cmp = element[0].expression_list()[0]
if not isinstance(lst_cmp, pr.ListComprehension):
raise IndexError
except IndexError:
r = list(
itertools.chain.from_iterable(
self.eval_statement(s) for s in element))
else:
r = [iterable.GeneratorComprehension(self, lst_cmp)]
call_path = element.generate_call_path()
next(call_path, None) # the first one has been used already
return self.follow_path(call_path, r, element.parent)
elif isinstance(element, pr.ListComprehension):
return self.eval_statement(element.stmt)
elif isinstance(element, pr.Lambda):
return [er.Function(self, element)]
# With things like params, these can also be functions...
elif isinstance(element, pr.Base) and element.isinstance(
er.Function, er.Class, er.Instance, iterable.ArrayInstance):
return [element]
# The string tokens are just operations (+, -, etc.)
elif isinstance(element, compiled.CompiledObject):
return [element]
elif isinstance(element, Token):
return []
else:
return self.eval_call(element)
def _parse_dot_name(self, pre_used_token=None):
"""
The dot name parser parses a name, variable or function and returns
their names.
:return: tuple of Name, next_token
"""
def append(el):
names.append(el)
self.module.temp_used_names.append(el[0])
names = []
tok = next(self._gen) if pre_used_token is None else pre_used_token
if tok.type != tokenize.NAME and tok.string != '*':
return None, tok
first_pos = tok.start_pos
append((tok.string, first_pos))
while True:
end_pos = tok.end_pos
tok = next(self._gen)
if tok.string != '.':
break
tok = next(self._gen)
if tok.type != tokenize.NAME:
break
append((tok.string, tok.start_pos))
n = pr.Name(self.module, names, first_pos, end_pos) if names else None
return n, tok
def _parse_function(self):
"""
The parser for a text functions. Process the tokens, which follow a
function definition.
:return: Return a Scope representation of the tokens.
:rtype: Function
"""
first_pos = self._gen.current.start_pos
tok = next(self._gen)
if tok.type != tokenize.NAME:
return None
fname, tok = self._parse_name(tok)
if tok.string != '(':
return None
params = self._parse_parentheses(is_class=False)
colon = next(self._gen)
annotation = None
if colon.string in ('-', '->'):
# parse annotations
if colon.string == '-':
# The Python 2 tokenizer doesn't understand this
colon = next(self._gen)
if colon.string != '>':
return None
annotation, colon = self._parse_statement(added_breaks=[':'])
if colon.string != ':':
return None
# Because of 2 line func param definitions
return pr.Function(self.module, fname, params, first_pos, annotation)
def _parse_class(self):
"""
The parser for a text class. Process the tokens, which follow a
class definition.
:return: Return a Scope representation of the tokens.
:rtype: Class
"""
first_pos = self._gen.current.start_pos
cname = next(self._gen)
if cname.type != tokenize.NAME:
debug.warning("class: syntax err, token is not a name@%s (%s: %s)",
cname.start_pos[0], tokenize.tok_name[cname.type],
cname.string)
return None
cname = pr.Name(self.module, [(cname.string, cname.start_pos)],
cname.start_pos, cname.end_pos)
super = []
_next = next(self._gen)
if _next.string == '(':
super = self._parse_parentheses()
_next = next(self._gen)
if _next.string != ':':
debug.warning("class syntax: %s@%s", cname, _next.start_pos[0])
return None
return pr.Class(self.module, cname, super, first_pos)
def _eval_statement_element(self, element):
if pr.Array.is_type(element, pr.Array.NOARRAY):
r = list(
itertools.chain.from_iterable(
self.eval_statement(s) for s in element))
call_path = element.generate_call_path()
next(call_path, None) # the first one has been used already
return self.follow_path(call_path, r, element.parent)
elif isinstance(element, pr.ListComprehension):
loop = _evaluate_list_comprehension(element)
# Caveat: parents are being changed, but this doesn't matter,
# because nothing else uses it.
element.stmt.parent = loop
return self.eval_statement(element.stmt)
elif isinstance(element, pr.Lambda):
return [er.Function(self, element)]
# With things like params, these can also be functions...
elif isinstance(element, pr.Base) and element.isinstance(
er.Function, er.Class, er.Instance, iterable.ArrayInstance):
return [element]
# The string tokens are just operations (+, -, etc.)
elif isinstance(element, compiled.CompiledObject):
return [element]
elif not isinstance(element, Token):
return self.eval_call(element)
else:
return []
def get_completions(user_stmt, bs):
if isinstance(user_stmt, pr.Import):
context = self._user_context.get_context()
next(context) # skip the path
if next(context) == 'from':
# completion is just "import" if before stands from ..
return ((k, bs) for k in keywords.keyword_names('import'))
return self._simple_complete(path, like)
def get_completions(user_stmt, bs):
if isinstance(user_stmt, pr.Import):
context = self._module.get_context()
next(context) # skip the path
if next(context) == 'from':
# completion is just "import" if before stands from ..
return ((k, bs) for k in keywords.keyword_names('import'))
return self._simple_complete(path, like)
def __next__(self):
if self._push_backs:
return self._push_backs.pop(0)
self.previous = self.current
self.current = next(self._tokenizer)
return self.current
def __next__(self):
if self._push_backs:
return self._push_backs.pop(0)
self.previous = self.current
self.current = next(self._tokenizer)
return self.current
def follow_call_path(path, scope, position):
"""Follows a path generated by `pr.Call.generate_call_path()`"""
current = next(path)
if isinstance(current, pr.Array):
result = [er.Array(current)]
else:
if isinstance(current, pr.NamePart):
# This is the first global lookup.
scopes = find_name(scope,
current,
position=position,
search_global=True)
else:
if current.type in (pr.Call.STRING, pr.Call.NUMBER):
t = type(current.name).__name__
scopes = find_name(builtin.Builtin.scope, t)
else:
debug.warning('unknown type:', current.type, current)
scopes = []
# Make instances of those number/string objects.
scopes = [er.Instance(s, (current.name, )) for s in scopes]
result = imports.strip_imports(scopes)
return follow_paths(path, result, scope, position=position)
def __next__(self):
""" Generate the next tokenize pattern. """
try:
typ, tok, start_pos, end_pos, self.parserline = next(self._gen)
# dedents shouldn't change positions
if typ != tokenize.DEDENT:
self.start_pos, self.end_pos = start_pos, end_pos
except (StopIteration, common.MultiLevelStopIteration):
# on finish, set end_pos correctly
s = self.scope
while s is not None:
if isinstance(s, pr.Module) \
and not isinstance(s, pr.SubModule):
self.module.end_pos = self.end_pos
break
s.end_pos = self.end_pos
s = s.parent
raise
if self.user_position and (self.start_pos[0] == self.user_position[0]
or self.user_scope is None
and self.start_pos[0] >= self.user_position[0]):
debug.dbg('user scope found [%s] = %s' %
(self.parserline.replace('\n', ''), repr(self.scope)))
self.user_scope = self.scope
self.last_token = self.current
self.current = (typ, tok)
return self.current
def __next__(self):
""" Generate the next tokenize pattern. """
try:
typ, tok, start_pos, end_pos, self.parserline = next(self._gen)
# dedents shouldn't change positions
if typ != tokenize.DEDENT:
self.start_pos = start_pos
if typ not in (tokenize.INDENT, tokenize.NEWLINE, tokenize.NL):
self.start_pos, self.end_pos = start_pos, end_pos
except (StopIteration, common.MultiLevelStopIteration):
# on finish, set end_pos correctly
s = self._scope
while s is not None:
if isinstance(s, pr.Module) \
and not isinstance(s, pr.SubModule):
self.module.end_pos = self.end_pos
break
s.end_pos = self.end_pos
s = s.parent
raise
if self.user_position and (self.start_pos[0] == self.user_position[0]
or self.user_scope is None and
self.start_pos[0] >= self.user_position[0]):
debug.dbg('user scope found [%s] = %s' %
(self.parserline.replace('\n', ''), repr(self._scope)))
self.user_scope = self._scope
self._current = typ, tok
return self._current
def eval_call_path(self, path, scope, position):
"""
Follows a path generated by `pr.StatementElement.generate_call_path()`.
"""
current = next(path)
if isinstance(current, pr.Array):
if current.type == pr.Array.NOARRAY:
try:
lst_cmp = current[0].expression_list()[0]
if not isinstance(lst_cmp, pr.ListComprehension):
raise IndexError
except IndexError:
types = list(chain.from_iterable(self.eval_statement(s)
for s in current))
else:
types = [iterable.GeneratorComprehension(self, lst_cmp)]
else:
types = [iterable.Array(self, current)]
else:
if isinstance(current, pr.Name):
# This is the first global lookup.
types = self.find_types(scope, current, position=position,
search_global=True)
else:
# for pr.Literal
types = [compiled.create(self, current.value)]
types = imports.follow_imports(self, types)
return self.follow_path(path, types, scope)
def goto_definitions(self):
"""
Return the definitions of a the path under the cursor. goto function!
This follows complicated paths and returns the end, not the first
definition. The big difference between :meth:`goto_assignments` and
:meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
follow imports and statements. Multiple objects may be returned,
because Python itself is a dynamic language, which means depending on
an option you can have two different versions of a function.
:rtype: list of :class:`api_classes.Definition`
"""
def resolve_import_paths(scopes):
for s in scopes.copy():
if isinstance(s, imports.ImportPath):
scopes.remove(s)
scopes.update(resolve_import_paths(set(s.follow())))
return scopes
goto_path = self._module.get_path_under_cursor()
context = self._module.get_context()
scopes = set()
lower_priority_operators = ("()", "(", ",")
"""Operators that could hide callee."""
if next(context) in ("class", "def"):
scopes = set([self._module.parser.user_scope])
elif not goto_path:
op = self._module.get_operator_under_cursor()
if op and op not in lower_priority_operators:
scopes = set([keywords.get_operator(op, self.pos)])
# Fetch definition of callee
if not goto_path:
(call, _) = self._func_call_and_param_index()
if call is not None:
while call.next is not None:
call = call.next
# reset cursor position:
(row, col) = call.name.end_pos
self.pos = (row, max(col - 1, 0))
self._module = modules.ModuleWithCursor(self._source_path, source=self.source, position=self.pos)
# then try to find the path again
goto_path = self._module.get_path_under_cursor()
if not scopes:
if goto_path:
scopes = set(self._prepare_goto(goto_path))
elif op in lower_priority_operators:
scopes = set([keywords.get_operator(op, self.pos)])
scopes = resolve_import_paths(scopes)
# add keywords
scopes |= keywords.get_keywords(string=goto_path, pos=self.pos)
d = set([api_classes.Definition(s) for s in scopes if not isinstance(s, imports.ImportPath._GlobalNamespace)])
return self._sorted_defs(d)
def _goto(self, add_import_name=False):
"""
Used for goto_assignments and usages.
:param add_import_name: Add the the name (if import) to the result.
"""
def follow_inexistent_imports(defs):
""" Imports can be generated, e.g. following
`multiprocessing.dummy` generates an import dummy in the
multiprocessing module. The Import doesn't exist -> follow.
"""
definitions = set(defs)
for d in defs:
if isinstance(d.parent, pr.Import) \
and d.start_pos == (0, 0):
i = imports.ImportPath(self._evaluator,
d.parent).follow(is_goto=True)
definitions.remove(d)
definitions |= follow_inexistent_imports(i)
return definitions
goto_path = self._user_context.get_path_under_cursor()
context = self._user_context.get_context()
user_stmt = self._parser.user_stmt()
if next(context) in ('class', 'def'):
user_scope = self._parser.user_scope()
definitions = set([user_scope.name])
search_name = unicode(user_scope.name)
elif isinstance(user_stmt, pr.Import):
s, name_part = helpers.get_on_import_stmt(self._evaluator,
self._user_context,
user_stmt)
try:
definitions = [s.follow(is_goto=True)[0]]
except IndexError:
definitions = []
search_name = unicode(name_part)
if add_import_name:
import_name = user_stmt.get_defined_names()
# imports have only one name
if not user_stmt.star \
and name_part == import_name[0].names[-1]:
definitions.append(import_name[0])
else:
stmt = self._get_under_cursor_stmt(goto_path)
defs, search_name = self._evaluator.goto(stmt)
definitions = follow_inexistent_imports(defs)
if isinstance(user_stmt, pr.Statement):
c = user_stmt.expression_list()
if c and not isinstance(c[0], (str, unicode)) \
and c[0].start_pos > self._pos \
and not re.search(r'\.\w+$', goto_path):
# The cursor must be after the start, otherwise the
# statement is just an assignee.
definitions = [user_stmt]
return definitions, search_name
def _defined_names(scope):
"""
List sub-definitions (e.g., methods in class).
:type scope: Scope
:rtype: list of Definition
"""
pair = next(evaluate.get_names_of_scope(scope, star_search=False, include_builtin=False), None)
names = pair[1] if pair else []
return [Definition(d) for d in sorted(names, key=lambda s: s.start_pos)]
def _goto(self, add_import_name=False):
"""
Used for goto_assignments and usages.
:param add_import_name: Add the the name (if import) to the result.
"""
def follow_inexistent_imports(defs):
""" Imports can be generated, e.g. following
`multiprocessing.dummy` generates an import dummy in the
multiprocessing module. The Import doesn't exist -> follow.
"""
definitions = set(defs)
for d in defs:
if isinstance(d.parent, pr.Import) \
and d.start_pos == (0, 0):
i = imports.ImportPath(self._evaluator, d.parent).follow(is_goto=True)
definitions.remove(d)
definitions |= follow_inexistent_imports(i)
return definitions
goto_path = self._user_context.get_path_under_cursor()
context = self._user_context.get_context()
user_stmt = self._parser.user_stmt()
if next(context) in ('class', 'def'):
user_scope = self._parser.user_scope()
definitions = set([user_scope.name])
search_name = unicode(user_scope.name)
elif isinstance(user_stmt, pr.Import):
s, name_part = helpers.get_on_import_stmt(self._evaluator,
self._user_context, user_stmt)
try:
definitions = [s.follow(is_goto=True)[0]]
except IndexError:
definitions = []
search_name = unicode(name_part)
if add_import_name:
import_name = user_stmt.get_defined_names()
# imports have only one name
if not user_stmt.star \
and name_part == import_name[0].names[-1]:
definitions.append(import_name[0])
else:
stmt = self._get_under_cursor_stmt(goto_path)
defs, search_name = self._evaluator.goto(stmt)
definitions = follow_inexistent_imports(defs)
if isinstance(user_stmt, pr.Statement):
c = user_stmt.expression_list()
if c and not isinstance(c[0], (str, unicode)) \
and c[0].start_pos > self._pos \
and not re.search(r'\.\w+$', goto_path):
# The cursor must be after the start, otherwise the
# statement is just an assignee.
definitions = [user_stmt]
return definitions, search_name
def _parse_import_list(self):
"""
The parser for the imports. Unlike the class and function parse
function, this returns no Import class, but rather an import list,
which is then added later on.
The reason, why this is not done in the same class lies in the nature
of imports. There are two ways to write them:
- from ... import ...
- import ...
To distinguish, this has to be processed after the parser.
:return: List of imports.
:rtype: list
"""
imports = []
brackets = False
continue_kw = [",", ";", "\n", '\r\n', ')'] \
+ list(set(keyword.kwlist) - set(['as']))
while True:
defunct = False
tok = next(self._gen)
if tok.string == '(': # python allows only one `(` in the statement.
brackets = True
tok = next(self._gen)
if brackets and tok.type == tokenize.NEWLINE:
tok = next(self._gen)
i, tok = self._parse_dot_name(tok)
if not i:
defunct = True
name2 = None
if tok.string == 'as':
name2, tok = self._parse_dot_name()
imports.append((i, name2, defunct))
while tok.string not in continue_kw:
tok = next(self._gen)
if not (tok.string == ","
or brackets and tok.type == tokenize.NEWLINE):
break
return imports
def defined_names(evaluator, scope):
"""
List sub-definitions (e.g., methods in class).
:type scope: Scope
:rtype: list of Definition
"""
pair = next(get_names_of_scope(evaluator, scope, star_search=False,
include_builtin=False), None)
names = pair[1] if pair else []
names = [n for n in names if isinstance(n, pr.Import) or (len(n) == 1)]
return [Definition(evaluator, d) for d in sorted(names, key=lambda s: s.start_pos)]
def _user_stmt(self, is_completion=False):
user_stmt = self._parser.user_stmt
debug.speed('parsed')
if is_completion and not user_stmt:
# for statements like `from x import ` (cursor not in statement)
pos = next(self._module.get_context(yield_positions=True))
last_stmt = pos and self._parser.module.get_statement_for_position(
pos, include_imports=True)
if isinstance(last_stmt, pr.Import):
user_stmt = last_stmt
return user_stmt
def _user_stmt(self, is_completion=False):
user_stmt = self._parser.user_stmt
debug.speed('parsed')
if is_completion and not user_stmt:
# for statements like `from x import ` (cursor not in statement)
pos = next(self._module.get_context(yield_positions=True))
last_stmt = pos and self._parser.module.get_statement_for_position(
pos, include_imports=True)
if isinstance(last_stmt, pr.Import):
user_stmt = last_stmt
return user_stmt
def _parse_import_list(self):
"""
The parser for the imports. Unlike the class and function parse
function, this returns no Import class, but rather an import list,
which is then added later on.
The reason, why this is not done in the same class lies in the nature
of imports. There are two ways to write them:
- from ... import ...
- import ...
To distinguish, this has to be processed after the parser.
:return: List of imports.
:rtype: list
"""
imports = []
brackets = False
continue_kw = [",", ";", "\n", '\r\n', ')'] \
+ list(set(keyword.kwlist) - set(['as']))
while True:
defunct = False
tok = next(self._gen)
if tok.string == '(': # python allows only one `(` in the statement.
brackets = True
tok = next(self._gen)
if brackets and tok.type == tokenize.NEWLINE:
tok = next(self._gen)
i, tok = self._parse_dot_name(tok)
if not i:
defunct = True
name2 = None
if tok.string == 'as':
name2, tok = self._parse_dot_name()
imports.append((i, name2, defunct))
while tok.string not in continue_kw:
tok = next(self._gen)
if not (tok.string == "," or brackets and tok.type == tokenize.NEWLINE):
break
return imports
def _goto(self, add_import_name=False):
"""
Used for goto_assignments and usages.
:param add_import_name: TODO add description
"""
def follow_inexistent_imports(defs):
""" Imports can be generated, e.g. following
`multiprocessing.dummy` generates an import dummy in the
multiprocessing module. The Import doesn't exist -> follow.
"""
definitions = set(defs)
for d in defs:
if isinstance(d.parent, pr.Import) \
and d.start_pos == (0, 0):
i = imports.ImportPath(d.parent).follow(is_goto=True)
definitions.remove(d)
definitions |= follow_inexistent_imports(i)
return definitions
goto_path = self._module.get_path_under_cursor()
context = self._module.get_context()
user_stmt = self._parser.user_stmt
if next(context) in ('class', 'def'):
user_scope = self._parser.user_scope
definitions = set([user_scope.name])
search_name = unicode(user_scope.name)
elif isinstance(user_stmt, pr.Import):
s, name_part = self._get_on_import_stmt()
try:
definitions = [s.follow(is_goto=True)[0]]
except IndexError:
definitions = []
search_name = unicode(name_part)
if add_import_name:
import_name = user_stmt.get_defined_names()
# imports have only one name
if name_part == import_name[0].names[-1]:
definitions.append(import_name[0])
else:
stmt = self._get_under_cursor_stmt(goto_path)
defs, search_name = evaluate.goto(stmt)
definitions = follow_inexistent_imports(defs)
if isinstance(user_stmt, pr.Statement):
call = user_stmt.get_commands()[0]
if not isinstance(call, (str, unicode)) and \
call.start_pos > self.pos:
# The cursor must be after the start, otherwise the
# statement is just an assignee.
definitions = [user_stmt]
return definitions, search_name
def _defined_names(scope):
"""
List sub-definitions (e.g., methods in class).
:type scope: Scope
:rtype: list of Definition
"""
pair = next(
evaluate.get_names_of_scope(scope,
star_search=False,
include_builtin=False), None)
names = pair[1] if pair else []
return [Definition(d) for d in sorted(names, key=lambda s: s.start_pos)]
def _create_params(function, lst):
if not lst:
return []
if is_node(lst[0], 'typedargslist', 'varargslist'):
params = []
iterator = iter(lst[0].children)
for n in iterator:
stars = 0
if n in ('*', '**'):
stars = len(n.value)
n = next(iterator)
op = next(iterator, None)
if op == '=':
default = next(iterator)
next(iterator, None)
else:
default = None
params.append(Param(n, function, default, stars))
return params
else:
return [Param(lst[0], function)]
def get_completions(user_stmt, bs):
# TODO this closure is ugly. it also doesn't work with
# simple_complete (used for Interpreter), somehow redo.
module = self._parser.module()
names, level, only_modules, unfinished_dotted = \
helpers.check_error_statements(module, self._pos)
completion_names = []
if names is not None:
imp_names = [n for n in names if n.end_pos < self._pos]
i = imports.get_importer(self._evaluator, imp_names, module,
level)
completion_names = i.completion_names(self._evaluator,
only_modules)
# TODO this paragraph is necessary, but not sure it works.
context = self._user_context.get_context()
if not next(context).startswith('.'): # skip the path
if next(context) == 'from':
# completion is just "import" if before stands from ..
if unfinished_dotted:
return completion_names
else:
return keywords.keyword_names('import')
if isinstance(user_stmt, pr.Import):
module = self._parser.module()
completion_names += imports.completion_names(
self._evaluator, user_stmt, self._pos)
return completion_names
if names is None and not isinstance(user_stmt, pr.Import):
if not path and not dot:
# add keywords
completion_names += keywords.keyword_names(all=True)
# TODO delete? We should search for valid parser
# transformations.
completion_names += self._simple_complete(path, dot, like)
return completion_names
def _goto(self, add_import_name=False):
"""
Used for goto_assignments and usages.
:param add_import_name: TODO add description
"""
def follow_inexistent_imports(defs):
""" Imports can be generated, e.g. following
`multiprocessing.dummy` generates an import dummy in the
multiprocessing module. The Import doesn't exist -> follow.
"""
definitions = set(defs)
for d in defs:
if isinstance(d.parent, pr.Import) \
and d.start_pos == (0, 0):
i = imports.ImportPath(d.parent).follow(is_goto=True)
definitions.remove(d)
definitions |= follow_inexistent_imports(i)
return definitions
goto_path = self._module.get_path_under_cursor()
context = self._module.get_context()
user_stmt = self._parser.user_stmt
if next(context) in ('class', 'def'):
user_scope = self._parser.user_scope
definitions = set([user_scope.name])
search_name = unicode(user_scope.name)
elif isinstance(user_stmt, pr.Import):
s, name_part = self._get_on_import_stmt()
try:
definitions = [s.follow(is_goto=True)[0]]
except IndexError:
definitions = []
search_name = unicode(name_part)
if add_import_name:
import_name = user_stmt.get_defined_names()
# imports have only one name
if name_part == import_name[0].names[-1]:
definitions.append(import_name[0])
else:
stmt = self._get_under_cursor_stmt(goto_path)
defs, search_name = evaluate.goto(stmt)
definitions = follow_inexistent_imports(defs)
if isinstance(user_stmt, pr.Statement):
if user_stmt.get_commands()[0].start_pos > self.pos:
# The cursor must be after the start, otherwise the
# statement is just an assignee.
definitions = [user_stmt]
return definitions, search_name
def goto_definitions(self):
"""
Return the definitions of a the path under the cursor. goto function!
This follows complicated paths and returns the end, not the first
definition. The big difference between :meth:`goto_assignments` and
:meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
follow imports and statements. Multiple objects may be returned,
because Python itself is a dynamic language, which means depending on
an option you can have two different versions of a function.
:rtype: list of :class:`classes.Definition`
"""
def resolve_import_paths(scopes):
for s in scopes.copy():
if isinstance(s, imports.ImportPath):
scopes.remove(s)
scopes.update(resolve_import_paths(set(s.follow())))
return scopes
user_stmt = self._parser.user_stmt_with_whitespace()
goto_path = self._user_context.get_path_under_cursor()
context = self._user_context.get_context()
definitions = set()
if next(context) in ('class', 'def'):
definitions = set([self._parser.user_scope()])
else:
# Fetch definition of callee, if there's no path otherwise.
if not goto_path:
(call,
_) = helpers.func_call_and_param_index(user_stmt, self._pos)
if call is not None:
while call.next is not None:
call = call.next
# reset cursor position:
(row, col) = call.name.end_pos
pos = (row, max(col - 1, 0))
self._user_context = UserContext(self.source, pos)
# then try to find the path again
goto_path = self._user_context.get_path_under_cursor()
if not definitions:
if goto_path:
definitions = set(self._prepare_goto(goto_path))
definitions = resolve_import_paths(definitions)
d = set([
classes.Definition(self._evaluator, s) for s in definitions
if s is not imports.ImportPath.GlobalNamespace
])
return helpers.sorted_definitions(d)
def _parse_function(self):
"""
The parser for a text functions. Process the tokens, which follow a
function definition.
:return: Return a Scope representation of the tokens.
:rtype: Function
"""
first_pos = self._gen.current.start_pos
tok = next(self._gen)
if tok.type != tokenize.NAME:
return None
fname = pr.Name(self.module, [(tok.string, tok.start_pos)],
tok.start_pos, tok.end_pos)
tok = next(self._gen)
if tok.string != '(':
return None
params = self._parse_parentheses()
colon = next(self._gen)
annotation = None
if colon.string in ('-', '->'):
# parse annotations
if colon.string == '-':
# The Python 2 tokenizer doesn't understand this
colon = next(self._gen)
if colon.string != '>':
return None
annotation, colon = self._parse_statement(added_breaks=[':'])
if colon.string != ':':
return None
# because of 2 line func param definitions
return pr.Function(self.module, fname, params, first_pos, annotation)
def get_completions(user_stmt, bs):
# TODO this closure is ugly. it also doesn't work with
# simple_complete (used for Interpreter), somehow redo.
module = self._parser.module()
names, level, only_modules, unfinished_dotted = \
helpers.check_error_statements(module, self._pos)
completion_names = []
if names is not None:
imp_names = [n for n in names if n.end_pos < self._pos]
i = imports.get_importer(self._evaluator, imp_names, module, level)
completion_names = i.completion_names(self._evaluator, only_modules)
# TODO this paragraph is necessary, but not sure it works.
context = self._user_context.get_context()
if not next(context).startswith('.'): # skip the path
if next(context) == 'from':
# completion is just "import" if before stands from ..
if unfinished_dotted:
return completion_names
else:
return keywords.keyword_names('import')
if isinstance(user_stmt, pr.Import):
module = self._parser.module()
completion_names += imports.completion_names(self._evaluator,
user_stmt, self._pos)
return completion_names
if names is None and not isinstance(user_stmt, pr.Import):
if not path and not dot:
# add keywords
completion_names += keywords.keyword_names(all=True)
# TODO delete? We should search for valid parser
# transformations.
completion_names += self._simple_complete(path, dot, like)
return completion_names
def goto_definitions(self):
"""
Return the definitions of a the path under the cursor. goto function!
This follows complicated paths and returns the end, not the first
definition. The big difference between :meth:`goto_assignments` and
:meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
follow imports and statements. Multiple objects may be returned,
because Python itself is a dynamic language, which means depending on
an option you can have two different versions of a function.
:rtype: list of :class:`classes.Definition`
"""
def resolve_import_paths(scopes):
for s in scopes.copy():
if isinstance(s, imports.ImportWrapper):
scopes.remove(s)
scopes.update(resolve_import_paths(set(s.follow())))
return scopes
goto_path = self._user_context.get_path_under_cursor()
context = self._user_context.get_context()
definitions = set()
if next(context) in ('class', 'def'):
definitions = set(
[er.wrap(self._evaluator, self._parser.user_scope())])
else:
# Fetch definition of callee, if there's no path otherwise.
if not goto_path:
definitions = set(signature._definition
for signature in self.call_signatures())
if re.match('\w[\w\d_]*$', goto_path) and not definitions:
user_stmt = self._parser.user_stmt()
if user_stmt is not None and user_stmt.type == 'expr_stmt':
for name in user_stmt.get_defined_names():
if name.start_pos <= self._pos <= name.end_pos:
# TODO scaning for a name and then using it should be
# the default.
definitions = set(
self._evaluator.goto_definition(name))
if not definitions and goto_path:
definitions = set(self._prepare_goto(goto_path))
definitions = resolve_import_paths(definitions)
names = [s.name for s in definitions]
defs = [classes.Definition(self._evaluator, name) for name in names]
return helpers.sorted_definitions(set(defs))
def __init__(self, source, module_path=None, no_docstr=False,
tokenizer=None, top_module=None):
self.no_docstr = no_docstr
tokenizer = tokenizer or tokenize.source_tokens(source)
self._gen = PushBackTokenizer(tokenizer)
# initialize global Scope
start_pos = next(self._gen).start_pos
self._gen.push_last_back()
self.module = pr.SubModule(module_path, start_pos, top_module)
self._scope = self.module
self._top_module = top_module or self.module
try:
self._parse()
except (common.MultiLevelStopIteration, StopIteration):
# StopIteration needs to be added as well, because python 2 has a
# strange way of handling StopIterations.
# sometimes StopIteration isn't catched. Just ignore it.
# on finish, set end_pos correctly
pass
s = self._scope
while s is not None:
s.end_pos = self._gen.current.end_pos
s = s.parent
# clean up unused decorators
for d in self._decorators:
# set a parent for unused decorators, avoid NullPointerException
# because of `self.module.used_names`.
d.parent = self.module
self.module.end_pos = self._gen.current.end_pos
if self._gen.current.type == tokenize.NEWLINE:
# This case is only relevant with the FastTokenizer, because
# otherwise there's always an ENDMARKER.
# we added a newline before, so we need to "remove" it again.
#
# NOTE: It should be keep end_pos as-is if the last token of
# a source is a NEWLINE, otherwise the newline at the end of
# a source is not included in a ParserNode.code.
if self._gen.previous.type != tokenize.NEWLINE:
self.module.end_pos = self._gen.previous.end_pos
del self._gen
def goto_definitions(self):
"""
Return the definitions of a the path under the cursor. goto function!
This follows complicated paths and returns the end, not the first
definition. The big difference between :meth:`goto_assignments` and
:meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
follow imports and statements. Multiple objects may be returned,
because Python itself is a dynamic language, which means depending on
an option you can have two different versions of a function.
:rtype: list of :class:`classes.Definition`
"""
def resolve_import_paths(scopes):
for s in scopes.copy():
if isinstance(s, imports.ImportPath):
scopes.remove(s)
scopes.update(resolve_import_paths(set(s.follow())))
return scopes
user_stmt = self._parser.user_stmt_with_whitespace()
goto_path = self._user_context.get_path_under_cursor()
context = self._user_context.get_context()
definitions = set()
if next(context) in ('class', 'def'):
definitions = set([self._parser.user_scope()])
else:
# Fetch definition of callee, if there's no path otherwise.
if not goto_path:
(call, _) = helpers.func_call_and_param_index(user_stmt, self._pos)
if call is not None:
while call.next is not None:
call = call.next
# reset cursor position:
(row, col) = call.name.end_pos
pos = (row, max(col - 1, 0))
self._user_context = UserContext(self.source, pos)
# then try to find the path again
goto_path = self._user_context.get_path_under_cursor()
if not definitions:
if goto_path:
definitions = set(self._prepare_goto(goto_path))
definitions = resolve_import_paths(definitions)
d = set([classes.Definition(self._evaluator, s) for s in definitions
if s is not imports.ImportPath.GlobalNamespace])
return helpers.sorted_definitions(d)
def goto_definitions(self):
"""
Return the definitions of a the path under the cursor. goto function!
This follows complicated paths and returns the end, not the first
definition. The big difference between :meth:`goto_assignments` and
:meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
follow imports and statements. Multiple objects may be returned,
because Python itself is a dynamic language, which means depending on
an option you can have two different versions of a function.
:rtype: list of :class:`classes.Definition`
"""
def resolve_import_paths(scopes):
for s in scopes.copy():
if isinstance(s, imports.ImportWrapper):
scopes.remove(s)
scopes.update(resolve_import_paths(set(s.follow())))
return scopes
goto_path = self._user_context.get_path_under_cursor()
context = self._user_context.get_context()
definitions = set()
if next(context) in ('class', 'def'):
definitions = set([er.wrap(self._evaluator, self._parser.user_scope())])
else:
# Fetch definition of callee, if there's no path otherwise.
if not goto_path:
definitions = set(signature._definition
for signature in self.call_signatures())
if re.match('\w[\w\d_]*$', goto_path) and not definitions:
user_stmt = self._parser.user_stmt()
if user_stmt is not None and user_stmt.type == 'expr_stmt':
for name in user_stmt.get_defined_names():
if name.start_pos <= self._pos <= name.end_pos:
# TODO scaning for a name and then using it should be
# the default.
definitions = set(self._evaluator.goto_definition(name))
if not definitions and goto_path:
definitions = set(self._prepare_goto(goto_path))
definitions = resolve_import_paths(definitions)
names = [s.name for s in definitions]
defs = [classes.Definition(self._evaluator, name) for name in names]
return helpers.sorted_definitions(set(defs))
def defined_names(evaluator, scope):
"""
List sub-definitions (e.g., methods in class).
:type scope: Scope
:rtype: list of Definition
"""
# Calling get_names_of_scope doesn't make sense always. It might include
# star imports or inherited stuff. Wanted?
# TODO discuss!
if isinstance(scope, pr.Module):
pair = scope, scope.get_defined_names()
else:
pair = next(get_names_of_scope(evaluator, scope, star_search=False,
include_builtin=False), None)
names = pair[1] if pair else []
return [Definition(evaluator, d) for d in sorted(names, key=lambda s: s.start_pos)]
def defined_names(evaluator, scope):
"""
List sub-definitions (e.g., methods in class).
:type scope: Scope
:rtype: list of Definition
"""
pair = next(
get_names_of_scope(evaluator,
scope,
star_search=False,
include_builtin=False), None)
names = pair[1] if pair else []
names = [n for n in names if isinstance(n, pr.Import) or (len(n) == 1)]
return [
Definition(evaluator, d)
for d in sorted(names, key=lambda s: s.start_pos)
]
def defined_names(evaluator, scope):
"""
List sub-definitions (e.g., methods in class).
:type scope: Scope
:rtype: list of Definition
"""
# Calling get_names_of_scope doesn't make sense always. It might include
# star imports or inherited stuff. Wanted?
# TODO discuss!
if isinstance(scope, pr.Module):
pair = scope, scope.get_defined_names()
else:
pair = next(get_names_of_scope(evaluator, scope, star_search=False,
include_builtin=False), None)
names = pair[1] if pair else []
names = [n for n in names if isinstance(n, pr.Import) or (len(n) == 1)]
return [Definition(evaluator, d) for d in sorted(names, key=lambda s: s.start_pos)]
def _follow_path(self, path, typ, scope):
"""
Uses a generator and tries to complete the path, e.g.::
foo.bar.baz
`_follow_path` is only responsible for completing `.bar.baz`, the rest
is done in the `follow_call` function.
"""
# current is either an Array or a Scope.
try:
current = next(path)
except StopIteration:
return None
debug.dbg('_follow_path: %s in scope %s', current, typ)
result = []
if isinstance(current, pr.Array):
# This must be an execution, either () or [].
if current.type == pr.Array.LIST:
if hasattr(typ, 'get_index_types'):
if isinstance(typ, compiled.CompiledObject):
# CompiledObject doesn't contain an evaluator instance.
result = typ.get_index_types(self, current)
else:
result = typ.get_index_types(current)
elif current.type not in [pr.Array.DICT]:
# Scope must be a class or func - make an instance or execution.
result = self.execute(typ, current)
else:
# Curly braces are not allowed, because they make no sense.
debug.warning('strange function call with {} %s %s', current,
typ)
else:
# The function must not be decorated with something else.
if typ.isinstance(er.Function):
typ = typ.get_magic_function_scope()
else:
# This is the typical lookup while chaining things.
if filter_private_variable(typ, scope, current):
return []
types = self.find_types(typ, current)
result = imports.follow_imports(self, types)
return self.follow_path(path, result, scope)
def _follow_path(self, path, typ, scope):
"""
Uses a generator and tries to complete the path, e.g.::
foo.bar.baz
`_follow_path` is only responsible for completing `.bar.baz`, the rest
is done in the `follow_call` function.
"""
# current is either an Array or a Scope.
try:
current = next(path)
except StopIteration:
return None
debug.dbg('_follow_path: %s in scope %s', current, typ)
result = []
if isinstance(current, pr.Array):
# This must be an execution, either () or [].
if current.type == pr.Array.LIST:
if hasattr(typ, 'get_index_types'):
if isinstance(typ, compiled.CompiledObject):
# CompiledObject doesn't contain an evaluator instance.
result = typ.get_index_types(self, current)
else:
result = typ.get_index_types(current)
elif current.type not in [pr.Array.DICT]:
# Scope must be a class or func - make an instance or execution.
result = self.execute(typ, current)
else:
# Curly braces are not allowed, because they make no sense.
debug.warning('strange function call with {} %s %s', current, typ)
else:
# The function must not be decorated with something else.
if typ.isinstance(er.Function):
typ = typ.get_magic_function_scope()
else:
# This is the typical lookup while chaining things.
if filter_private_variable(typ, scope, current):
return []
types = self.find_types(typ, current)
result = imports.follow_imports(self, types)
return self.follow_path(path, result, scope)
def eval_call_path(self, path, scope, position):
"""
Follows a path generated by `pr.StatementElement.generate_call_path()`.
"""
current = next(path)
if isinstance(current, pr.Array):
types = [iterable.Array(self, current)]
else:
if isinstance(current, pr.NamePart):
# This is the first global lookup.
types = self.find_types(scope, current, position=position,
search_global=True)
else:
# for pr.Literal
types = [compiled.create(current.value)]
types = imports.strip_imports(self, types)
return self.follow_path(path, types, scope)
def eval_call_path(self, path, scope, position):
"""
Follows a path generated by `pr.StatementElement.generate_call_path()`.
"""
current = next(path)
if isinstance(current, pr.Array):
types = [iterable.Array(self, current)]
else:
if isinstance(current, pr.NamePart):
# This is the first global lookup.
types = self.find_types(scope, current, position=position,
search_global=True)
else:
# for pr.Literal
types = [compiled.create(self, current.value)]
types = imports.follow_imports(self, types)
return self.follow_path(path, types, scope)
def follow_call_path(path, scope, position):
"""Follows a path generated by `pr.StatementElement.generate_call_path()`"""
current = next(path)
if isinstance(current, pr.Array):
result = [er.Array(current)]
else:
if isinstance(current, pr.NamePart):
# This is the first global lookup.
scopes = find_name(scope, current, position=position,
search_global=True)
else:
# for pr.Literal
scopes = find_name(builtin.Builtin.scope, current.type_as_string())
# Make instances of those number/string objects.
scopes = [er.Instance(s, (current.value,)) for s in scopes]
result = imports.strip_imports(scopes)
return follow_paths(path, result, scope, position=position)
def goto_definitions(self):
"""
Return the definitions of a the path under the cursor. goto function!
This follows complicated paths and returns the end, not the first
definition. The big difference between :meth:`goto_assignments` and
:meth:`goto_definitions` is that :meth:`goto_assignments` doesn't
follow imports and statements. Multiple objects may be returned,
because Python itself is a dynamic language, which means depending on
an option you can have two different versions of a function.
:rtype: list of :class:`classes.Definition`
"""
def resolve_import_paths(scopes):
for s in scopes.copy():
if isinstance(s, imports.ImportWrapper):
scopes.remove(s)
scopes.update(resolve_import_paths(set(s.follow())))
return scopes
user_stmt = self._parser.user_stmt_with_whitespace()
goto_path = self._user_context.get_path_under_cursor()
context = self._user_context.get_context()
definitions = set()
if next(context) in ('class', 'def'):
definitions = set([er.wrap(self._evaluator, self._parser.user_scope())])
else:
# Fetch definition of callee, if there's no path otherwise.
if not goto_path:
call, _, _ = search_call_signatures(user_stmt, self._pos)
if call is not None:
definitions = set(self._evaluator.eval_call(call))
if not definitions:
if goto_path:
definitions = set(self._prepare_goto(goto_path))
definitions = resolve_import_paths(definitions)
names = [s.name for s in definitions
if s is not imports.ImportWrapper.GlobalNamespace]
defs = [classes.Definition(self._evaluator, name) for name in names]
return helpers.sorted_definitions(set(defs))
def __next__(self):
if self.closed:
raise MultiLevelStopIteration()
try:
self.current = next(self.gen)
except tokenize.TokenError:
# We just ignore this error, I try to handle it earlier - as
# good as possible
debug.warning("parentheses not closed error")
return self.__next__()
except IndentationError:
# This is an error, that tokenize may produce, because the code
# is not indented as it should. Here it just ignores this line
# and restarts the parser.
# (This is a rather unlikely error message, for normal code,
# tokenize seems to be pretty tolerant)
debug.warning("indentation error on line %s, ignoring it" % self.current[2][0])
# add the starting line of the last position
self.offset = (self.offset[0] + self.current[2][0], self.current[2][1])
self.gen = PushBackIterator(tokenize.generate_tokens(self.readline))
return self.__next__()
c = list(self.current)
# stop if a new class or definition is started at position zero.
breaks = ["def", "class", "@"]
if self.stop_on_scope and c[1] in breaks and c[2][1] == 0:
if self.first_scope:
self.closed = True
raise MultiLevelStopIteration()
elif c[1] != "@":
self.first_scope = True
if self.first:
c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1]
c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1]
self.first = False
else:
c[2] = self.offset[0] + c[2][0], c[2][1]
c[3] = self.offset[0] + c[3][0], c[3][1]
return c
def follow_path(path, scope, call_scope, position=None):
"""
Uses a generator and tries to complete the path, e.g.::
foo.bar.baz
`follow_path` is only responsible for completing `.bar.baz`, the rest is
done in the `follow_call` function.
"""
# current is either an Array or a Scope.
try:
current = next(path)
except StopIteration:
return None
debug.dbg('follow %s in scope %s' % (current, scope))
result = []
if isinstance(current, pr.Array):
# This must be an execution, either () or [].
if current.type == pr.Array.LIST:
if hasattr(scope, 'get_index_types'):
result = scope.get_index_types(current)
elif current.type not in [pr.Array.DICT]:
# Scope must be a class or func - make an instance or execution.
debug.dbg('exe', scope)
result = er.Execution(scope, current).get_return_types()
else:
# Curly braces are not allowed, because they make no sense.
debug.warning('strange function call with {}', current, scope)
else:
# The function must not be decorated with something else.
if scope.isinstance(er.Function):
scope = scope.get_magic_method_scope()
else:
# This is the typical lookup while chaining things.
if filter_private_variable(scope, call_scope, current):
return []
result = imports.strip_imports(find_name(scope, current,
position=position))
return follow_paths(path, set(result), call_scope, position=position)
def follow_path(path, scope, call_scope, position=None):
"""
Uses a generator and tries to complete the path, e.g.::
foo.bar.baz
`follow_path` is only responsible for completing `.bar.baz`, the rest is
done in the `follow_call` function.
"""
# current is either an Array or a Scope.
try:
current = next(path)
except StopIteration:
return None
debug.dbg('follow %s in scope %s' % (current, scope))
result = []
if isinstance(current, pr.Array):
# This must be an execution, either () or [].
if current.type == pr.Array.LIST:
if hasattr(scope, 'get_index_types'):
result = scope.get_index_types(current)
elif current.type not in [pr.Array.DICT]:
# Scope must be a class or func - make an instance or execution.
debug.dbg('exe', scope)
result = er.Execution(scope, current).get_return_types()
else:
# Curly braces are not allowed, because they make no sense.
debug.warning('strange function call with {}', current, scope)
else:
# The function must not be decorated with something else.
if scope.isinstance(er.Function):
scope = scope.get_magic_method_scope()
else:
# This is the typical lookup while chaining things.
if filter_private_variable(scope, call_scope, current):
return []
result = imports.strip_imports(find_name(scope, current,
position=position))
return follow_paths(path, set(result), call_scope, position=position)
def __next__(self):
""" Generate the next tokenize pattern. """
try:
typ, tok, self.start_pos, self.end_pos, \
self.parserline = next(self._gen)
except (StopIteration, common.MultiLevelStopIteration):
# on finish, set end_pos correctly
s = self.scope
while s is not None:
s.end_pos = self.end_pos
s = s.parent
raise
if self.user_position and (self.start_pos[0] == self.user_position[0]
or self.user_scope is None
and self.start_pos[0] >= self.user_position[0]):
debug.dbg('user scope found [%s] = %s' % \
(self.parserline.replace('\n', ''), repr(self.scope)))
self.user_scope = self.scope
self.last_token = self.current
self.current = (typ, tok)
return self.current
def follow_call_path(path, scope, position):
"""Follows a path generated by `pr.Call.generate_call_path()`"""
current = next(path)
if isinstance(current, pr.Array):
result = [er.Array(current)]
else:
if isinstance(current, pr.NamePart):
# This is the first global lookup.
scopes = find_name(scope, current, position=position,
search_global=True)
else:
if current.type in (pr.Call.STRING, pr.Call.NUMBER):
t = type(current.name).__name__
scopes = find_name(builtin.Builtin.scope, t)
else:
debug.warning('unknown type:', current.type, current)
scopes = []
# Make instances of those number/string objects.
scopes = [er.Instance(s, (current.name,)) for s in scopes]
result = imports.strip_imports(scopes)
return follow_paths(path, result, scope, position=position)
def _get_on_import_stmt(self, user_stmt, is_like_search=False):
""" Resolve the user statement, if it is an import. Only resolve the
parts until the user position. """
import_names = user_stmt.get_all_import_names()
kill_count = -1
cur_name_part = None
for i in import_names:
if user_stmt.alias == i:
continue
for name_part in i.names:
if name_part.end_pos >= self._pos:
if not cur_name_part:
cur_name_part = name_part
kill_count += 1
context = self._module.get_context()
just_from = next(context) == 'from'
i = imports.ImportPath(user_stmt,
is_like_search,
kill_count=kill_count,
direct_resolve=True,
is_just_from=just_from)
return i, cur_name_part
def __next__(self):
if self.pushes:
self.current = self.pushes.pop()
else:
self.current = next(self.iterator)
return self.current
def __next__(self):
if self.closed:
raise MultiLevelStopIteration()
if self.push_backs:
return self.push_backs.pop(0)
self.last_previous = self.previous
self.previous = self.current
self.current = next(self.gen)
c = list(self.current)
if c[0] == tokenize.ENDMARKER:
self.current = self.previous
self.previous = self.last_previous
raise MultiLevelStopIteration()
# this is exactly the same check as in fast_parser, but this time with
# tokenize and therefore precise.
breaks = ['def', 'class', '@']
if self.is_first:
c[2] = self.offset[0] + c[2][0], self.offset[1] + c[2][1]
c[3] = self.offset[0] + c[3][0], self.offset[1] + c[3][1]
self.is_first = False
else:
c[2] = self.offset[0] + c[2][0], c[2][1]
c[3] = self.offset[0] + c[3][0], c[3][1]
self.current = c
def close():
if not self.first_stmt:
self.closed = True
raise MultiLevelStopIteration()
# ignore indents/comments
if self.is_fast_parser \
and self.previous[0] in (tokenize.INDENT, tokenize.NL, None,
tokenize.NEWLINE, tokenize.DEDENT) \
and c[0] not in (tokenize.COMMENT, tokenize.INDENT,
tokenize.NL, tokenize.NEWLINE, tokenize.DEDENT):
# print c, tokenize.tok_name[c[0]]
tok = c[1]
indent = c[2][1]
if indent < self.parser_indent: # -> dedent
self.parser_indent = indent
self.new_indent = False
if not self.in_flow or indent < self.old_parser_indent:
close()
self.in_flow = False
elif self.new_indent:
self.parser_indent = indent
self.new_indent = False
if not self.in_flow:
if tok in FLOWS or tok in breaks:
self.in_flow = tok in FLOWS
if not self.is_decorator and not self.in_flow:
close()
self.is_decorator = '@' == tok
if not self.is_decorator:
self.old_parser_indent = self.parser_indent
self.parser_indent += 1 # new scope: must be higher
self.new_indent = True
if tok != '@':
if self.first_stmt and not self.new_indent:
self.parser_indent = indent
self.first_stmt = False
return c
def _parse(self):
"""
The main part of the program. It analyzes the given code-text and
returns a tree-like scope. For a more detailed description, see the
class description.
:param text: The code which should be parsed.
:param type: str
:raises: IndentationError
"""
extended_flow = ['else', 'elif', 'except', 'finally']
statement_toks = ['{', '[', '(', '`']
self._decorators = []
self.freshscope = True
for tok in self._gen:
token_type = tok.type
tok_str = tok.string
first_pos = tok.start_pos
self.module.temp_used_names = []
# debug.dbg('main: tok=[%s] type=[%s] indent=[%s]', \
# tok, tokenize.tok_name[token_type], start_position[0])
# check again for unindented stuff. this is true for syntax
# errors. only check for names, because thats relevant here. If
# some docstrings are not indented, I don't care.
while first_pos[1] <= self._scope.start_pos[1] \
and (token_type == tokenize.NAME or tok_str in ('(', '['))\
and self._scope != self.module:
self._scope.end_pos = first_pos
self._scope = self._scope.parent
if isinstance(self._scope, pr.Module) \
and not isinstance(self._scope, pr.SubModule):
self._scope = self.module
if isinstance(self._scope, pr.SubModule):
use_as_parent_scope = self._top_module
else:
use_as_parent_scope = self._scope
if tok_str == 'def':
func = self._parse_function()
if func is None:
debug.warning("function: syntax error@%s", first_pos[0])
continue
self.freshscope = True
self._scope = self._scope.add_scope(func, self._decorators)
self._decorators = []
elif tok_str == 'class':
cls = self._parse_class()
if cls is None:
debug.warning("class: syntax error@%s" % first_pos[0])
continue
self.freshscope = True
self._scope = self._scope.add_scope(cls, self._decorators)
self._decorators = []
# import stuff
elif tok_str == 'import':
imports = self._parse_import_list()
for count, (m, alias, defunct) in enumerate(imports):
e = (alias or m or self._gen.previous).end_pos
end_pos = self._gen.previous.end_pos if count + 1 == len(
imports) else e
i = pr.Import(self.module,
first_pos,
end_pos,
m,
alias,
defunct=defunct)
self._check_user_stmt(i)
self._scope.add_import(i)
if not imports:
i = pr.Import(self.module,
first_pos,
self._gen.current.end_pos,
None,
defunct=True)
self._check_user_stmt(i)
self.freshscope = False
elif tok_str == 'from':
defunct = False
# take care for relative imports
relative_count = 0
while True:
tok = next(self._gen)
if tok.string != '.':
break
relative_count += 1
# the from import
mod, tok = self._parse_dot_name(self._gen.current)
tok_str = tok.string
if str(mod) == 'import' and relative_count:
self._gen.push_last_back()
tok_str = 'import'
mod = None
if not mod and not relative_count or tok_str != "import":
debug.warning("from: syntax error@%s", tok.start_pos[0])
defunct = True
if tok_str != 'import':
self._gen.push_last_back()
names = self._parse_import_list()
for count, (name, alias, defunct2) in enumerate(names):
star = name is not None and unicode(name.names[0]) == '*'
if star:
name = None
e = (alias or name or self._gen.previous).end_pos
end_pos = self._gen.previous.end_pos if count + 1 == len(
names) else e
i = pr.Import(self.module,
first_pos,
end_pos,
name,
alias,
mod,
star,
relative_count,
defunct=defunct or defunct2)
self._check_user_stmt(i)
self._scope.add_import(i)
self.freshscope = False
# loops
elif tok_str == 'for':
set_stmt, tok = self._parse_statement(added_breaks=['in'],
names_are_set_vars=True)
if tok.string != 'in':
debug.warning('syntax err, for flow incomplete @%s',
tok.start_pos[0])
try:
statement, tok = self._parse_statement()
except StopIteration:
statement, tok = None, None
s = [] if statement is None else [statement]
f = pr.ForFlow(self.module, s, first_pos, set_stmt)
self._scope = self._scope.add_statement(f)
if tok is None or tok.string != ':':
debug.warning('syntax err, for flow started @%s',
first_pos[0])
elif tok_str in ['if', 'while', 'try', 'with'] + extended_flow:
added_breaks = []
command = tok_str
if command in ('except', 'with'):
added_breaks.append(',')
# multiple inputs because of with
inputs = []
first = True
while first or command == 'with' and tok.string not in (
':', '\n', '\r\n'):
statement, tok = \
self._parse_statement(added_breaks=added_breaks)
if command == 'except' and tok.string == ',':
# the except statement defines a var
# this is only true for python 2
n, tok = self._parse_dot_name()
if n:
n.parent = statement
statement.as_names.append(n)
if statement:
inputs.append(statement)
first = False
f = pr.Flow(self.module, command, inputs, first_pos)
if command in extended_flow:
# the last statement has to be another part of
# the flow statement, because a dedent releases the
# main scope, so just take the last statement.
try:
s = self._scope.statements[-1].set_next(f)
except (AttributeError, IndexError):
# If set_next doesn't exist, just add it.
s = self._scope.add_statement(f)
else:
s = self._scope.add_statement(f)
self._scope = s
if tok.string != ':':
debug.warning('syntax err, flow started @%s',
tok.start_pos[0])
# returns
elif tok_str in ('return', 'yield'):
s = tok.start_pos
self.freshscope = False
# add returns to the scope
func = self._scope.get_parent_until(pr.Function)
if tok_str == 'yield':
func.is_generator = True
stmt, tok = self._parse_statement()
if stmt is not None:
stmt.parent = use_as_parent_scope
try:
func.returns.append(stmt)
# start_pos is the one of the return statement
stmt.start_pos = s
except AttributeError:
debug.warning('return in non-function')
elif tok_str == 'assert':
stmt, tok = self._parse_statement()
if stmt is not None:
stmt.parent = use_as_parent_scope
self._scope.asserts.append(stmt)
elif tok_str in STATEMENT_KEYWORDS:
stmt, _ = self._parse_statement()
kw = pr.KeywordStatement(tok_str, tok.start_pos,
use_as_parent_scope, stmt)
self._scope.add_statement(kw)
if stmt is not None and tok_str == 'global':
for t in stmt._token_list:
if isinstance(t, pr.Name):
# Add the global to the top module, it counts there.
self.module.add_global(t)
# decorator
elif tok_str == '@':
stmt, tok = self._parse_statement()
if stmt is not None:
self._decorators.append(stmt)
elif tok_str == 'pass':
continue
# default
elif token_type in (tokenize.NAME, tokenize.STRING,
tokenize.NUMBER, tokenize.OP) \
or tok_str in statement_toks:
# this is the main part - a name can be a function or a
# normal var, which can follow anything. but this is done
# by the statement parser.
stmt, tok = self._parse_statement(self._gen.current)
if stmt:
self._scope.add_statement(stmt)
self.freshscope = False
else:
if token_type not in (tokenize.COMMENT, tokenize.NEWLINE,
tokenize.ENDMARKER):
debug.warning('Token not used: %s %s %s', tok_str,
tokenize.tok_name[token_type], first_pos)
continue
self.no_docstr = False
def _parse_statement(self,
pre_used_token=None,
added_breaks=None,
stmt_class=pr.Statement,
names_are_set_vars=False):
"""
Parses statements like::
a = test(b)
a += 3 - 2 or b
and so on. One line at a time.
:param pre_used_token: The pre parsed token.
:type pre_used_token: set
:return: Statement + last parsed token.
:rtype: (Statement, str)
"""
set_vars = []
level = 0 # The level of parentheses
if pre_used_token:
tok = pre_used_token
else:
tok = next(self._gen)
while tok.type == tokenize.COMMENT:
# remove newline and comment
next(self._gen)
tok = next(self._gen)
first_pos = tok.start_pos
opening_brackets = ['{', '(', '[']
closing_brackets = ['}', ')', ']']
# the difference between "break" and "always break" is that the latter
# will even break in parentheses. This is true for typical flow
# commands like def and class and the imports, which will never be used
# in a statement.
breaks = set(['\n', '\r\n', ':', ')'])
always_break = [
';', 'import', 'from', 'class', 'def', 'try', 'except', 'finally',
'while', 'return', 'yield'
]
not_first_break = ['del', 'raise']
if added_breaks:
breaks |= set(added_breaks)
tok_list = []
as_names = []
in_lambda_param = False
while not (tok.string in always_break or tok.string in not_first_break
and not tok_list or tok.string in breaks and level <= 0
and not (in_lambda_param and tok.string in ',:')):
try:
# print 'parse_stmt', tok, tokenize.tok_name[token_type]
is_kw = tok.string in OPERATOR_KEYWORDS
if tok.type == tokenize.OP or is_kw:
tok_list.append(pr.Operator(tok.string, tok.start_pos))
else:
tok_list.append(tok)
if tok.string == 'as':
tok = next(self._gen)
if tok.type == tokenize.NAME:
n, tok = self._parse_dot_name(self._gen.current)
if n:
set_vars.append(n)
as_names.append(n)
tok_list.append(n)
continue
elif tok.string == 'lambda':
breaks.discard(':')
in_lambda_param = True
elif in_lambda_param and tok.string == ':':
in_lambda_param = False
elif tok.type == tokenize.NAME and not is_kw:
n, tok = self._parse_dot_name(self._gen.current)
# removed last entry, because we add Name
tok_list.pop()
if n:
tok_list.append(n)
continue
elif tok.string in opening_brackets:
level += 1
elif tok.string in closing_brackets:
level -= 1
tok = next(self._gen)
except (StopIteration, common.MultiLevelStopIteration):
# comes from tokenizer
break
if not tok_list:
return None, tok
first_tok = tok_list[0]
# docstrings
if len(tok_list) == 1 and isinstance(first_tok, tokenize.Token) \
and first_tok.type == tokenize.STRING:
# Normal docstring check
if self.freshscope and not self.no_docstr:
self._scope.add_docstr(first_tok)
return None, tok
# Attribute docstring (PEP 224) support (sphinx uses it, e.g.)
# If string literal is being parsed...
elif first_tok.type == tokenize.STRING:
with common.ignored(IndexError, AttributeError):
# ...then set it as a docstring
self._scope.statements[-1].add_docstr(first_tok)
return None, tok
stmt = stmt_class(self.module,
tok_list,
first_pos,
tok.end_pos,
as_names=as_names,
names_are_set_vars=names_are_set_vars)
stmt.parent = self._top_module
self._check_user_stmt(stmt)
if tok.string in always_break + not_first_break:
self._gen.push_last_back()
return stmt, tok