def description(self):
"""
A description of the :class:`.Definition` object, which is heavily used
in testing. e.g. for ``isinstance`` it returns ``def isinstance``.
"""
d = self.definition
if isinstance(d, evaluate.InstanceElement):
d = d.var
if isinstance(d, evaluate.parsing.Name):
d = d.parent
if isinstance(d, evaluate.Array):
d = 'class ' + d.type
elif isinstance(d, (parsing.Class, evaluate.Class, evaluate.Instance)):
d = 'class ' + unicode(d.name)
elif isinstance(d, (evaluate.Function, evaluate.parsing.Function)):
d = 'def ' + unicode(d.name)
elif isinstance(d, evaluate.parsing.Module):
# only show module name
d = 'module %s' % self.module_name
elif self.is_keyword:
d = 'keyword %s' % d.name
else:
d = d.get_code().replace('\n', '')
return d
def description(self):
"""
A description of the :class:`.Definition` object, which is heavily used
in testing. e.g. for ``isinstance`` it returns ``def isinstance``.
"""
d = self.definition
if isinstance(d, evaluate.InstanceElement):
d = d.var
if isinstance(d, evaluate.parsing.Name):
d = d.parent
if isinstance(d, evaluate.Array):
d = 'class ' + d.type
elif isinstance(d, (parsing.Class, evaluate.Class, evaluate.Instance)):
d = 'class ' + unicode(d.name)
elif isinstance(d, (evaluate.Function, evaluate.parsing.Function)):
d = 'def ' + unicode(d.name)
elif isinstance(d, evaluate.parsing.Module):
# only show module name
d = 'module %s' % self.module_name
elif self.is_keyword:
d = 'keyword %s' % d.name
else:
d = d.get_code().replace('\n', '')
return d
def test_multibyte_script(self):
""" `jedi.Script` must accept multi-byte string source. """
try:
code = unicode("import datetime; datetime.d")
comment = utf8("# multi-byte comment あいうえおä")
s = (unicode('%s\n%s') % (code, comment)).encode('utf-8')
except NameError:
pass # python 3 has no unicode method
else:
assert len(self.complete(s, (1, len(code))))
def test_multibyte_script(self):
""" `jedi.Script` must accept multi-byte string source. """
try:
code = unicode("import datetime; datetime.d")
comment = utf8("# multi-byte comment あいうえおä")
s = (unicode('%s\n%s') % (code, comment)).encode('utf-8')
except NameError:
pass # python 3 has no unicode method
else:
assert len(self.complete(s, (1, len(code))))
def _goto(self, add_import_name=False):
"""
Used for goto and related_names.
:param add_import_name: TODO add description
"""
def follow_inexistent_imports(defs):
""" Imports can be generated, e.g. following
`multiprocessing.dummy` generates an import dummy in the
multiprocessing module. The Import doesn't exist -> follow.
"""
definitions = set(defs)
for d in defs:
if isinstance(d.parent, parsing.Import) \
and d.start_pos == (0, 0):
i = imports.ImportPath(d.parent).follow(is_goto=True)
definitions.remove(d)
definitions |= follow_inexistent_imports(i)
return definitions
goto_path = self._module.get_path_under_cursor()
context = self._module.get_context()
user_stmt = self._parser.user_stmt
if next(context) in ('class', 'def'):
user_scope = self._parser.user_scope
definitions = set([user_scope.name])
search_name = unicode(user_scope.name)
elif isinstance(user_stmt, parsing.Import):
s, name_part = self._get_on_import_stmt()
try:
definitions = [s.follow(is_goto=True)[0]]
except IndexError:
definitions = []
search_name = unicode(name_part)
if add_import_name:
import_name = user_stmt.get_defined_names()
# imports have only one name
if name_part == import_name[0].names[-1]:
definitions.append(import_name[0])
else:
stmt = self._get_under_cursor_stmt(goto_path)
defs, search_name = evaluate.goto(stmt)
definitions = follow_inexistent_imports(defs)
if isinstance(user_stmt, parsing.Statement):
if user_stmt.get_assignment_calls().start_pos > self.pos:
# The cursor must be after the start, otherwise the
# statement is just an assignee.
definitions = [user_stmt]
return definitions, search_name
def _goto(self, add_import_name=False):
"""
Used for goto and related_names.
:param add_import_name: TODO add description
"""
def follow_inexistent_imports(defs):
""" Imports can be generated, e.g. following
`multiprocessing.dummy` generates an import dummy in the
multiprocessing module. The Import doesn't exist -> follow.
"""
definitions = set(defs)
for d in defs:
if isinstance(d.parent, parsing.Import) \
and d.start_pos == (0, 0):
i = imports.ImportPath(d.parent).follow(is_goto=True)
definitions.remove(d)
definitions |= follow_inexistent_imports(i)
return definitions
goto_path = self._module.get_path_under_cursor()
context = self._module.get_context()
user_stmt = self._parser.user_stmt
if next(context) in ('class', 'def'):
user_scope = self._parser.user_scope
definitions = set([user_scope.name])
search_name = unicode(user_scope.name)
elif isinstance(user_stmt, parsing.Import):
s, name_part = self._get_on_import_stmt()
try:
definitions = [s.follow(is_goto=True)[0]]
except IndexError:
definitions = []
search_name = unicode(name_part)
if add_import_name:
import_name = user_stmt.get_defined_names()
# imports have only one name
if name_part == import_name[0].names[-1]:
definitions.append(import_name[0])
else:
stmt = self._get_under_cursor_stmt(goto_path)
defs, search_name = evaluate.goto(stmt)
definitions = follow_inexistent_imports(defs)
if isinstance(user_stmt, parsing.Statement):
if user_stmt.get_assignment_calls().start_pos > self.pos:
# The cursor must be after the start, otherwise the
# statement is just an assignee.
definitions = [user_stmt]
return definitions, search_name
def related_names(self, additional_module_paths=[]):
"""
Return :class:`api_classes.RelatedName` objects, which contain all
names that point to the definition of the name under the cursor. This
is very useful for refactoring (renaming), or to show all usages of a
variable.
.. todo:: Implement additional_module_paths
:rtype: list of :class:`api_classes.RelatedName`
"""
user_stmt = self._parser.user_stmt
definitions, search_name = self._goto(add_import_name=True)
if isinstance(user_stmt, parsing.Statement) \
and self.pos < user_stmt.get_assignment_calls().start_pos:
# the search_name might be before `=`
definitions = [v for v in user_stmt.set_vars
if unicode(v.names[-1]) == search_name]
if not isinstance(user_stmt, parsing.Import):
# import case is looked at with add_import_name option
definitions = dynamic.related_name_add_import_modules(definitions,
search_name)
module = set([d.get_parent_until() for d in definitions])
module.add(self._parser.module)
names = dynamic.related_names(definitions, search_name, module)
for d in set(definitions):
if isinstance(d, parsing.Module):
names.append(api_classes.RelatedName(d, d))
else:
names.append(api_classes.RelatedName(d.names[-1], d))
return sorted(set(names), key=lambda x: (x.module_path, x.start_pos),
reverse=True)
def __init__(self, definition, start_pos):
self.start_pos = start_pos
self.definition = definition
self.is_keyword = isinstance(definition, keywords.Keyword)
# generate a path to the definition
self.module_path = unicode(definition.get_parent_until().path)
def source_to_unicode(source, encoding=None):
def detect_encoding():
""" For the implementation of encoding definitions in Python, look at:
http://www.python.org/dev/peps/pep-0263/
http://docs.python.org/2/reference/lexical_analysis.html#encoding-\
declarations
"""
if encoding is not None:
return encoding
if source.startswith('\xef\xbb\xbf'):
# UTF-8 byte-order mark
return 'utf-8'
first_two_lines = re.match(r'(?:[^\n]*\n){0,2}', source).group(0)
possible_encoding = re.match("coding[=:]\s*([-\w.]+)", first_two_lines)
if possible_encoding:
return possible_encoding.group(1)
else:
# the default if nothing else has been set -> PEP 263
return 'iso-8859-1'
if isinstance(source, unicode):
# only cast str/bytes
return source
# cast to unicode by default
return unicode(source, detect_encoding(), 'replace')
def source_to_unicode(source, encoding=None):
def detect_encoding():
""" For the implementation of encoding definitions in Python, look at:
http://www.python.org/dev/peps/pep-0263/
http://docs.python.org/2/reference/lexical_analysis.html#encoding-\
declarations
"""
if encoding is not None:
return encoding
if source.startswith('\xef\xbb\xbf'):
# UTF-8 byte-order mark
return 'utf-8'
first_two_lines = re.match(r'(?:[^\n]*\n){0,2}', source).group(0)
possible_encoding = re.match("coding[=:]\s*([-\w.]+)", first_two_lines)
if possible_encoding:
return possible_encoding.group(1)
else:
# the default if nothing else has been set -> PEP 263
return 'iso-8859-1'
if isinstance(source, unicode):
# only cast str/bytes
return source
# cast to unicode by default
return unicode(source, detect_encoding(), 'replace')
def related_names(self, additional_module_paths=[]):
"""
Returns `dynamic.RelatedName` objects, which contain all names, that
are defined by the same variable, function, class or import.
This function can be used either to show all the usages of a variable
or for renaming purposes.
TODO implement additional_module_paths
"""
user_stmt = self.parser.user_stmt
definitions, search_name = self._goto(add_import_name=True)
if isinstance(user_stmt, parsing.Statement) \
and self.pos < user_stmt.get_assignment_calls().start_pos:
# the search_name might be before `=`
definitions = [v for v in user_stmt.set_vars
if unicode(v) == search_name]
if not isinstance(user_stmt, parsing.Import):
# import case is looked at with add_import_name option
definitions = dynamic.related_name_add_import_modules(definitions,
search_name)
module = set([d.get_parent_until() for d in definitions])
module.add(self.parser.module)
names = dynamic.related_names(definitions, search_name, module)
for d in set(definitions):
if isinstance(d, parsing.Module):
names.append(api_classes.RelatedName(d, d))
else:
names.append(api_classes.RelatedName(d.names[0], d))
return sorted(set(names), key=lambda x: (x.module_path, x.start_pos),
reverse=True)
def __init__(self, definition, start_pos):
self.start_pos = start_pos
self.definition = definition
self.is_keyword = isinstance(definition, keywords.Keyword)
# generate a path to the definition
self.module_path = unicode(definition.get_parent_until().path)
def word(self):
""" In contrary to `complete` returns the whole word, e.g.
>>> isinstan
would return 'isinstance'.
"""
return unicode(self.name.names[-1])
def related_names(self, additional_module_paths=[]):
"""
Return :class:`api_classes.RelatedName` objects, which contain all
names that point to the definition of the name under the cursor. This
is very useful for refactoring (renaming), or to show all usages of a
variable.
.. todo:: Implement additional_module_paths
:rtype: list of :class:`api_classes.RelatedName`
"""
user_stmt = self._parser.user_stmt
definitions, search_name = self._goto(add_import_name=True)
if isinstance(user_stmt, parsing.Statement) \
and self.pos < user_stmt.get_assignment_calls().start_pos:
# the search_name might be before `=`
definitions = [v for v in user_stmt.set_vars
if unicode(v.names[-1]) == search_name]
if not isinstance(user_stmt, parsing.Import):
# import case is looked at with add_import_name option
definitions = dynamic.related_name_add_import_modules(definitions,
search_name)
module = set([d.get_parent_until() for d in definitions])
module.add(self._parser.module)
names = dynamic.related_names(definitions, search_name, module)
for d in set(definitions):
if isinstance(d, parsing.Module):
names.append(api_classes.RelatedName(d, d))
else:
names.append(api_classes.RelatedName(d.names[-1], d))
return sorted(set(names), key=lambda x: (x.module_path, x.start_pos))
def _get_id3(self, *keys):
if self._id3 is None:
return ''
for k in keys:
result = self._id3.get(k, None)
if result is not None:
break
return unicode(result or '')
def word(self):
"""
Similar to :meth:`Completion.complete`, but return the whole word, e.g. ::
>>> isinstan
would return 'isinstance'.
"""
return unicode(self.name.names[-1])
def word(self):
"""
Similar to :meth:`Completion.complete`, but return the whole word, e.g. ::
>>> isinstan
would return 'isinstance'.
"""
return unicode(self.name.names[-1])
def fetch(self, artist, song, album):
if not artist or not song:
return None # wikia needs both informations
r = requests.get(self.url % (artist, song))
if r.status_code != 200:
return None
# The api returns a pseudo json object, that contains a url.
match = re.search("'url':'([^']+)'", r.text)
if match is None:
return None
html_url = match.group(1)
debug.debug('fetch url', html_url)
if 'action=edit' in html_url:
return None
r = requests.get(html_url)
gracenote = False
if r.status_code != 200:
# try it also with Gracenote: (e.g. Glen Hansard - High Hope)
html_url = html_url[:9] + \
html_url[9:].replace('/', '/Gracenote:', 1)
debug.debug('fetch url', html_url)
r = requests.get(html_url)
gracenote = True
if r.status_code != 200:
return None
match = re.search(r"<div class='lyricbox'>", r.text)
#with open('/home/david/test.txt', 'w') as f:
# f.write(r.text.encode('UTF-8'))
if match is None:
debug.debug('src not found in url', html_url)
return None
# parse the result
soup = BeautifulSoup(r.text)
lyricbox = soup.find('div', "lyricbox")
if lyricbox is None:
debug.debug("BeautifulSoup doesn't find content", html_url)
return None
if gracenote:
# gracenote lyrics are in a separate paragraph
lyricbox = lyricbox.find('p')
lyrics = ''
for c in lyricbox.contents:
text = unicode(c).strip()
if type(c) == NavigableString:
lyrics += text.strip()
elif text.startswith('<br'):
lyrics += '\n'
return lyrics.strip()
def __init__(self, source, line, column, source_path,
source_encoding='utf-8'):
debug.reset_time()
try:
source = unicode(source, source_encoding, 'replace')
# Use 'replace' over 'ignore' to hold code structure.
except TypeError: # `source` is already a unicode object
pass
self.pos = line, column
self.module = modules.ModuleWithCursor(source_path, source=source,
position=self.pos)
self.source_path = source_path
debug.speed('init')
def full_name(self):
"""The path to a certain class/function, see #61."""
path = [unicode(p) for p in self.path]
# TODO add further checks, the mapping should only occur on stdlib.
try:
path[0] = self._mapping[path[0]]
except KeyError:
pass
for key, repl in self._tuple_mapping.items():
if tuple(path[:len(key)]) == key:
path = [repl] + path[len(key):]
return '.'.join(path if path[0] else path[1:])
def description(self):
""" Provides a description of the completion object
TODO return value is just __repr__ of some objects, improve! """
parent = self.name.parent
if parent is None:
return ''
t = self.type
if t == 'Statement' or t == 'Import':
desc = self.definition.get_code(False)
else:
desc = '.'.join(unicode(p) for p in self.path)
line_nr = '' if self.in_builtin_module else '@%s' % self.line_nr
return '%s: %s%s' % (t, desc, line_nr)
def test_unicode_script(self):
""" normally no unicode objects are being used. (<=2.7) """
s = unicode("import datetime; datetime.timedelta")
completions = self.complete(s)
assert len(completions)
assert type(completions[0].description) is unicode
s = utf8("author='öä'; author")
completions = self.complete(s)
assert type(completions[0].description) is unicode
s = utf8("#-*- coding: iso-8859-1 -*-\nauthor='öä'; author")
s = s.encode('latin-1')
completions = self.complete(s)
assert type(completions[0].description) is unicode
def test_unicode_script(self):
""" normally no unicode objects are being used. (<=2.7) """
s = unicode("import datetime; datetime.timedelta")
completions = self.complete(s)
assert len(completions)
assert type(completions[0].description) is unicode
s = utf8("author='öä'; author")
completions = self.complete(s)
assert type(completions[0].description) is unicode
s = utf8("#-*- coding: iso-8859-1 -*-\nauthor='öä'; author")
s = s.encode('latin-1')
completions = self.complete(s)
assert type(completions[0].description) is unicode
def full_name(self):
"""The path to a certain class/function, see #61."""
path = [unicode(p) for p in self.path]
# TODO add further checks, the mapping should only occur on stdlib.
if not path:
return None # for keywords the path is empty
try:
path[0] = self._mapping[path[0]]
except KeyError:
pass
for key, repl in self._tuple_mapping.items():
if tuple(path[:len(key)]) == key:
path = [repl] + path[len(key):]
return '.'.join(path if path[0] else path[1:])
def description(self):
"""
Provide a description of the completion object.
.. todo:: return value is just __repr__ of some objects, improve!
"""
parent = self.name.parent
if parent is None:
return ''
t = self.type
if t == 'Statement' or t == 'Import':
desc = self.definition.get_code(False)
else:
desc = '.'.join(unicode(p) for p in self.path)
line = '' if self.in_builtin_module else '@%s' % self.line
return '%s: %s%s' % (t, desc, line)
def __init__(self,
source,
line,
column,
source_path,
source_encoding='utf-8'):
debug.reset_time()
try:
source = unicode(source, source_encoding, 'replace')
# Use 'replace' over 'ignore' to hold code structure.
except TypeError: # `source` is already a unicode object
pass
self.pos = line, column
self.module = modules.ModuleWithCursor(source_path,
source=source,
position=self.pos)
self.source_path = source_path
debug.speed('init')
def run_test(source, f_name, lines_to_execute):
"""
This is the completion test for some cases. The tests are not unit test
like, they are rather integration tests.
It uses comments to specify a test in the next line. The comment also says,
which results are expected. The comment always begins with `#?`. The last
row symbolizes the cursor.
For example:
>>> #? ['ab']
>>> ab = 3; a
>>> #? int()
>>> ab = 3; ab
"""
def get_defs(correct, correct_start, path):
def defs(line_nr, indent):
script = api.Script(source, line_nr, indent, path)
return set(script.get_definition())
should_be = set()
number = 0
for index in re.finditer('(?: +|$)', correct):
if correct == ' ':
continue
# -1 for the comment, +3 because of the comment start `#? `
start = index.start()
if print_debug:
api.set_debug_function(None)
number += 1
try:
should_be |= defs(line_nr - 1, start + correct_start)
except Exception:
print('could not resolve %s indent %s' % (line_nr - 1, start))
raise
if print_debug:
api.set_debug_function(debug.print_to_stdout)
# because the objects have different ids, `repr` it, then compare it.
should_str = set(r.desc_with_module for r in should_be)
if len(should_str) < number:
raise Exception('Solution @%s not right, too few test results: %s'
% (line_nr - 1, should_str))
return should_str
fails = 0
tests = 0
correct = None
test_type = None
start = None
for line_nr, line in enumerate(StringIO(source)):
line = unicode(line)
line_nr += 1
if correct:
r = re.match('^(\d+)\s*(.*)$', correct)
if r:
index = int(r.group(1))
correct = r.group(2)
start += r.regs[2][0] # second group, start index
else:
index = len(line) - 1 # -1 for the \n
# if a list is wanted, use the completion test, otherwise the
# get_definition test
path = completion_test_dir + os.path.sep + f_name
try:
script = api.Script(source, line_nr, index, path)
if test_type == '!':
fails += run_goto_test(script, correct, line_nr)
elif test_type == '<':
fails += run_related_name_test(script, correct, line_nr)
elif correct.startswith('['):
fails += run_completion_test(script, correct, line_nr)
else:
should_str = get_defs(correct, start, path)
fails += run_definition_test(script, should_str, line_nr)
except Exception:
print(traceback.format_exc())
print('test @%s: %s' % (line_nr - 1, line))
fails += 1
correct = None
tests += 1
else:
try:
r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]+)', line)
# test_type is ? for completion and ! for goto
test_type = r.group(1)
correct = r.group(2)
start = r.start()
except AttributeError:
correct = None
else:
# reset the test, if only one specific test is wanted
if lines_to_execute and line_nr not in lines_to_execute:
correct = None
return tests, fails
def description(self):
"""A textual description of the object."""
return unicode(self.definition)
def __repr__(self):
if not self.artist:
return unicode(self.file_name)
return "<%s: %s - %s>" % (self.__class__.__name__, self.artist,
self.song)
def run_test(source, f_name, lines_to_execute):
"""
This is the completion test for some cases. The tests are not unit test
like, they are rather integration tests.
It uses comments to specify a test in the next line. The comment also says,
which results are expected. The comment always begins with `#?`. The last
row symbolizes the cursor.
For example:
>>> #? ['ab']
>>> ab = 3; a
>>> #? int()
>>> ab = 3; ab
"""
def get_defs(correct, correct_start, path):
def defs(line_nr, indent):
script = api.Script(source, line_nr, indent, path)
return set(script.get_definition())
should_be = set()
number = 0
for index in re.finditer('(?: +|$)', correct):
if correct == ' ':
continue
# -1 for the comment, +3 because of the comment start `#? `
start = index.start()
if print_debug:
api.set_debug_function(None)
number += 1
try:
should_be |= defs(line_nr - 1, start + correct_start)
except Exception:
raise Exception('could not resolve %s indent %s'
% (line_nr - 1, start))
if print_debug:
api.set_debug_function(debug.print_to_stdout)
# because the objects have different ids, `repr` it, then compare it.
should_str = set(r.desc_with_module for r in should_be)
if len(should_str) < number:
raise Exception('Solution @%s not right, too few test results: %s'
% (line_nr - 1, should_str))
return should_str
fails = 0
tests = 0
correct = None
test_type = None
start = None
for line_nr, line in enumerate(StringIO(source)):
line = unicode(line)
line_nr += 1
if correct:
r = re.match('^(\d+)\s*(.*)$', correct)
if r:
index = int(r.group(1))
correct = r.group(2)
start += r.regs[2][0] # second group, start index
else:
index = len(line) - 1 # -1 for the \n
# if a list is wanted, use the completion test, otherwise the
# get_definition test
path = completion_test_dir + os.path.sep + f_name
try:
script = api.Script(source, line_nr, index, path)
if test_type == '!':
fails += run_goto_test(script, correct, line_nr)
elif test_type == '<':
fails += run_related_name_test(script, correct, line_nr)
elif correct.startswith('['):
fails += run_completion_test(script, correct, line_nr)
else:
should_str = get_defs(correct, start, path)
fails += run_definition_test(script, should_str, line_nr)
except Exception:
print(traceback.format_exc())
print('test @%s: %s' % (line_nr - 1, line))
fails += 1
correct = None
tests += 1
else:
try:
r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]+)', line)
# test_type is ? for completion and ! for goto
test_type = r.group(1)
correct = r.group(2)
start = r.start()
except AttributeError:
correct = None
else:
# reset the test, if only one specific test is wanted
if lines_to_execute and line_nr not in lines_to_execute:
correct = None
return tests, fails
def description(self):
return unicode(self.definition)
def call_name(self):
""" The name (e.g. 'isinstance') as a string. """
return unicode(self.executable.name)
def __init__(self, name_part, scope):
super(RelatedName, self).__init__(scope, name_part.start_pos)
self.name_part = name_part
self.text = unicode(name_part)
self.end_pos = name_part.end_pos
def __init__(self, name_part, scope):
super(RelatedName, self).__init__(scope, name_part.start_pos)
self.name_part = name_part
self.text = unicode(name_part)
self.end_pos = name_part.end_pos
def raw_doc(self):
"""The raw docstring ``__doc__`` for any object."""
try:
return unicode(self.definition.docstr)
except AttributeError:
return ''
def description(self):
"""A textual description of the object."""
return unicode(self.definition)
def raw_doc(self):
"""The raw docstring ``__doc__`` for any object."""
try:
return unicode(self.definition.docstr)
except AttributeError:
return ''
def call_name(self):
""" The name (e.g. 'isinstance') as a string. """
return unicode(self.executable.name)
