Beispiel #1
0
    def _get_parser(self, code, parser_code, line_offset, nodes, no_docstr):
        h = hash(code)
        hashes = [n.hash for n in nodes]
        node = None
        try:
            index = hashes.index(h)
            if nodes[index].code != code:
                raise ValueError()
        except ValueError:
            p = Parser(parser_code, self.module_path,
                               self.user_position, offset=(line_offset, 0),
                               is_fast_parser=True, top_module=self.module,
                               no_docstr=no_docstr)
            p.module.parent = self.module
        else:
            if nodes[index] != self.current_node:
                offset = int(nodes[0] == self.current_node)
                self.current_node.old_children.pop(index - offset)
            node = nodes.pop(index)
            p = node.parser
            m = p.module
            m.line_offset += line_offset + 1 - m.start_pos[0]
            if self.user_position is not None and \
                    m.start_pos[0] <= self.user_position[0] <= m.end_pos[0]:
                # It's important to take care of the whole user
                # positioning stuff, if no reparsing is being done.
                p.user_stmt = m.get_statement_for_position(
                    self.user_position, include_imports=True)
                if p.user_stmt:
                    p.user_scope = p.user_stmt.parent
                else:
                    p.user_scope = self._scan_user_scope(m) or m

        return p, node
Beispiel #2
0
def test_carriage_return_statements():
    source = u(dedent('''
        foo = 'ns1!'

        # this is a namespace package
    '''))
    source = source.replace('\n', '\r\n')
    stmt = Parser(load_grammar(), source).module.statements[0]
    assert '#' not in stmt.get_code()
Beispiel #3
0
def test_carriage_return_statements():
    source = u(dedent('''
        foo = 'ns1!'

        # this is a namespace package
    '''))
    source = source.replace('\n', '\r\n')
    stmt = Parser(source).module.statements[0]
    assert '#' not in stmt.get_code()
Beispiel #4
0
def test_user_statement_on_import():
    """github #285"""
    s = "from datetime import (\n" \
        "    time)"

    for pos in [(2, 1), (2, 4)]:
        u = Parser(s, user_position=pos).user_stmt
        assert isinstance(u, pr.Import)
        assert u.defunct is False
        assert [str(n) for n in u.get_defined_names()] == ['time']
Beispiel #5
0
def test_module():
    module = Parser(load_grammar(), u('asdf'), 'example.py').module
    name = module.name
    assert str(name) == 'example'
    assert name.start_pos == (1, 0)
    assert name.end_pos == (1, 7)

    module = Parser(load_grammar(), u('asdf')).module
    name = module.name
    assert str(name) == ''
    assert name.start_pos == (1, 0)
    assert name.end_pos == (1, 0)
Beispiel #6
0
def test_module():
    module = Parser('asdf', 'example.py', no_docstr=True).module
    name = module.name
    assert str(name) == 'example'
    assert name.start_pos == (0, 0)
    assert name.end_pos == (0, 0)

    module = Parser('asdf', no_docstr=True).module
    name = module.name
    assert str(name) == ''
    assert name.start_pos == (0, 0)
    assert name.end_pos == (0, 0)
Beispiel #7
0
def assert_params(param_string, **wanted_dct):
    source = dedent('''
    def x(%s):
        pass
    ''') % param_string

    parser = Parser(load_grammar(), dedent(source))
    funcdef = parser.get_parsed_node().subscopes[0]
    dct = dict((p.name.value, p.default and p.default.get_code())
               for p in funcdef.params)
    assert dct == wanted_dct
    assert parser.get_parsed_node().get_code() == source
Beispiel #8
0
def test_carriage_return_statements():
    source = u(
        dedent(
            """
        foo = 'ns1!'

        # this is a namespace package
    """
        )
    )
    source = source.replace("\n", "\r\n")
    stmt = Parser(load_grammar(), source).module.statements[0]
    assert "#" not in stmt.get_code()
Beispiel #9
0
    def _get_parser(self, code, parser_code, line_offset, nodes, no_docstr):
        h = hash(code)
        hashes = [n.hash for n in nodes]
        node = None
        try:
            index = hashes.index(h)
            if nodes[index].code != code:
                raise ValueError()
        except ValueError:
            tokenizer = FastTokenizer(parser_code, line_offset)
            p = Parser(parser_code,
                       self.module_path,
                       tokenizer=tokenizer,
                       top_module=self.module,
                       no_docstr=no_docstr)
            p.module.parent = self.module
        else:
            if nodes[index] != self.current_node:
                offset = int(nodes[0] == self.current_node)
                self.current_node.old_children.pop(index - offset)
            node = nodes.pop(index)
            p = node.parser
            m = p.module
            m.line_offset += line_offset + 1 - m.start_pos[0]

        return p, node
def test_explicit_absolute_imports():
    """
    Detect modules with ``from __future__ import absolute_import``.
    """
    parser = Parser(load_grammar(),
                    u("from __future__ import absolute_import"), "test.py")
    assert parser.module.has_explicit_absolute_import
Beispiel #11
0
 def test_end_pos(self):
     # jedi issue #150
     s = u("x()\nx( )\nx(  )\nx (  )")
     parser = Parser(s)
     for i, s in enumerate(parser.module.statements, 3):
         for c in s.expression_list():
             self.assertEqual(c.execution.end_pos[1], i)
Beispiel #12
0
def test_equals(source):
    evaluator = Evaluator(load_grammar())
    node = Parser(load_grammar(), source, 'eval_input').get_parsed_node()
    results = evaluator.eval_element(node)
    assert len(results) == 1
    first = results.pop()
    assert isinstance(first, CompiledObject) and first.obj is True
Beispiel #13
0
def _load_faked_module(module):
    module_name = module.__name__
    if module_name == '__builtin__' and not is_py3:
        module_name = 'builtins'

    try:
        return modules[module_name]
    except KeyError:
        path = os.path.dirname(os.path.abspath(__file__))
        try:
            with open(os.path.join(path, 'fake', module_name) + '.pym') as f:
                source = f.read()
        except IOError:
            modules[module_name] = None
            return
        grammar = load_grammar('grammar3.4')
        module = Parser(grammar, unicode(source), module_name).module
        modules[module_name] = module

        if module_name == 'builtins' and not is_py3:
            # There are two implementations of `open` for either python 2/3.
            # -> Rename the python2 version (`look at fake/builtins.pym`).
            open_func = search_scope(module, 'open')
            open_func.children[1] = FakeName('open_python3')
            open_func = search_scope(module, 'open_python2')
            open_func.children[1] = FakeName('open')
        return module
Beispiel #14
0
 def test_end_pos_one_line(self):
     parsed = Parser(load_grammar(), dedent(u('''
     def testit():
         a = "huhu"
     ''')))
     tok = parsed.module.subscopes[0].statements[0].children[2]
     assert tok.end_pos == (3, 14)
def sys_path_with_modifications(evaluator, module):
    if module.path is None:
        # Support for modules without a path is bad, therefore return the
        # normal path.
        return list(get_sys_path())

    curdir = os.path.abspath(os.curdir)
    with common.ignored(OSError):
        os.chdir(os.path.dirname(module.path))

    result = _check_module(evaluator, module)
    result += _detect_django_path(module.path)
    # buildout scripts often contain the same sys.path modifications
    # the set here is used to avoid duplicate sys.path entries
    buildout_paths = set()
    for module_path in _get_buildout_scripts(module.path):
        try:
            with open(module_path, 'rb') as f:
                source = f.read()
        except IOError:
            pass
        else:
            p = Parser(evaluator.grammar, common.source_to_unicode(source),
                       module_path)
            for path in _check_module(p.module):
                if path not in buildout_paths:
                    buildout_paths.add(path)
                    result.append(path)
    # cleanup, back to old directory
    os.chdir(curdir)
    return list(result)
Beispiel #16
0
    def _get_node(self, source, parser_code, line_offset, nodes, no_docstr):
        """
        Side effect: Alters the list of nodes.
        """
        h = hash(source)
        for index, node in enumerate(nodes):
            #print('EQ', node, repr(node.source), repr(source))
            if node.hash == h and node.source == source:
                node.reset_node()
                nodes.remove(node)
                break
        else:
            tokenizer = FastTokenizer(parser_code)
            self.number_parsers_used += 1
            #print('CODE', repr(source))
            p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer)
            node = ParserNode(self.module)

            end = line_offset + p.module.end_pos[0]
            used_lines = self._lines[line_offset:end - 1]
            code_part_actually_used = ''.join(used_lines)
            node.set_parser(p, code_part_actually_used)

        self.current_node.add_node(node, line_offset)
        return node
Beispiel #17
0
    def _get_node(self, source, parser_code, line_offset, nodes):
        """
        Side effect: Alters the list of nodes.
        """
        indent = len(source) - len(source.lstrip('\t '))
        self.current_node = self.current_node.parent_until_indent(indent)

        h = hash(source)
        for index, node in enumerate(nodes):
            if node.hash == h and node.source == source:
                node.reset_node()
                nodes.remove(node)
                break
        else:
            tokenizer = FastTokenizer(parser_code)
            self.number_parsers_used += 1
            p = Parser(self._grammar,
                       parser_code,
                       self.module_path,
                       tokenizer=tokenizer)

            end = line_offset + p.module.end_pos[0]
            used_lines = self._lines[line_offset:end - 1]
            code_part_actually_used = ''.join(used_lines)

            node = ParserNode(self.module, p, code_part_actually_used)

        self.current_node.add_node(node, line_offset)
        return node
Beispiel #18
0
def _evaluate_for_statement_string(evaluator, string, module):
    code = dedent("""
    def pseudo_docstring_stuff():
        # Create a pseudo function for docstring statements.
    %s
    """)
    if string is None:
        return []

    for element in re.findall('((?:\w+\.)*\w+)\.', string):
        # Try to import module part in dotted name.
        # (e.g., 'threading' in 'threading.Thread').
        string = 'import %s\n' % element + string

    # Take the default grammar here, if we load the Python 2.7 grammar here, it
    # will be impossible to use `...` (Ellipsis) as a token. Docstring types
    # don't need to conform with the current grammar.
    p = Parser(load_grammar(), code % indent_block(string))
    try:
        pseudo_cls = p.module.subscopes[0]
        # First pick suite, then simple_stmt (-2 for DEDENT) and then the node,
        # which is also not the last item, because there's a newline.
        stmt = pseudo_cls.children[-1].children[-2].children[-2]
    except (AttributeError, IndexError):
        return []

    # Use the module of the param.
    # TODO this module is not the module of the param in case of a function
    # call. In that case it's the module of the function call.
    # stuffed with content from a function call.
    pseudo_cls.parent = module
    return list(_execute_types_in_stmt(evaluator, stmt))
 def test_end_pos(self):
     # jedi issue #150
     s = "x()\nx( )\nx(  )\nx (  )"
     parser = Parser(s)
     for i, s in enumerate(parser.module.statements, 3):
         for c in s.get_commands():
             self.assertEqual(c.execution.end_pos[1], i)
Beispiel #20
0
def test_get_code():
    """Use the same code that the parser also generates, to compare"""
    s = u('''"""a docstring"""
class SomeClass(object, mixin):
    def __init__(self):
        self.xy = 3.0
        """statement docstr"""
    def some_method(self):
        return 1
    def yield_method(self):
        while hasattr(self, 'xy'):
            yield True
        for x in [1, 2]:
            yield x
    def empty(self):
        pass
class Empty:
    pass
class WithDocstring:
    """class docstr"""
    pass
def method_with_docstring():
    """class docstr"""
    pass
''')
    assert Parser(load_grammar(), s).module.get_code() == s
Beispiel #21
0
def _evaluate_for_statement_string(evaluator, string, module):
    code = dedent("""
    def pseudo_docstring_stuff():
        '''Create a pseudo function for docstring statements.'''
    %s
    """)
    if string is None:
        return []

    for element in re.findall('((?:\w+\.)*\w+)\.', string):
        # Try to import module part in dotted name.
        # (e.g., 'threading' in 'threading.Thread').
        string = 'import %s\n' % element + string

    p = Parser(code % indent_block(string), no_docstr=True)
    pseudo_cls = p.module.subscopes[0]
    try:
        stmt = pseudo_cls.statements[-1]
    except IndexError:
        return []

    # Use the module of the param.
    # TODO this module is not the module of the param in case of a function
    # call. In that case it's the module of the function call.
    # stuffed with content from a function call.
    pseudo_cls.parent = module
    definitions = evaluator.eval_statement(stmt)
    it = (evaluator.execute(d) for d in definitions)
    # TODO Executing tuples does not make sense, people tend to say
    # `(str, int)` in a type annotation, which means that it returns a tuple
    # with both types.
    # At this point we just return the classes if executing wasn't possible,
    # i.e. is a tuple.
    return list(chain.from_iterable(it)) or definitions
def test_dont_break_imports_without_namespaces():
    """
    The code checking for ``from __future__ import absolute_import`` shouldn't
    assume that all imports have non-``None`` namespaces.
    """
    src = u("from __future__ import absolute_import\nimport xyzzy")
    parser = Parser(load_grammar(), src, "test.py")
    assert parser.module.has_explicit_absolute_import
Beispiel #23
0
 def test_end_pos_multi_line(self):
     parsed = Parser(load_grammar(), dedent(u('''
     def testit():
         a = """huhu
     asdfasdf""" + "h"
     ''')))
     tok = parsed.module.subscopes[0].statements[0].children[2].children[0]
     assert tok.end_pos == (4, 11)
Beispiel #24
0
def test_path_from_invalid_sys_path_assignment():
    SRC = u("""
import sys
sys.path = 'invalid'""")
    p = Parser(SRC)
    paths = _check_module(p.module)
    assert len(paths) > 0
    assert 'invalid' not in paths
Beispiel #25
0
def test_sys_path_with_modifications():
    SRC = dedent(u("""
        import os
    """))
    grammar = load_grammar()
    p = Parser(grammar, SRC)
    p.module.path = os.path.abspath(os.path.join(os.curdir, 'module_name.py'))
    paths = sys_path_with_modifications(Evaluator(grammar), p.module)
    assert '/tmp/.buildout/eggs/important_package.egg' in paths
def test_path_from_invalid_sys_path_assignment():
    SRC = dedent(u("""
        import sys
        sys.path = 'invalid'"""))
    grammar = load_grammar()
    p = Parser(grammar, SRC)
    paths = _check_module(Evaluator(grammar), p.module)
    assert len(paths) > 0
    assert 'invalid' not in paths
Beispiel #27
0
 def _parser(self):
     cache.invalidate_star_import_cache(self._path)
     if self._use_fast_parser:
         parser = FastParser(self._grammar, self._source, self._path)
         # Don't pickle that module, because the main module is changing quickly
         cache.save_parser(self._path, None, parser, pickling=False)
     else:
         parser = Parser(self._grammar, self._source, self._path)
     return parser
Beispiel #28
0
 def _get_under_cursor_stmt(self, cursor_txt):
     offset = self._line - 1, self._column
     r = Parser(cursor_txt, no_docstr=True, offset=offset)
     try:
         stmt = r.module.statements[0]
     except IndexError:
         raise NotFoundError()
     stmt.parent = self._parser.user_scope
     return stmt
Beispiel #29
0
def test_end_pos():
    s = u(dedent('''
                 x = ['a', 'b', 'c']
                 def func():
                     y = None
                 '''))
    parser = Parser(load_grammar(), s)
    scope = parser.module.subscopes[0]
    assert scope.start_pos == (3, 0)
    assert scope.end_pos == (5, 0)
Beispiel #30
0
 def check(src, result):
     # Python 2 tuple params should be ignored for now.
     grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
     m = Parser(grammar, u(src)).module
     if is_py3:
         assert not m.subscopes
     else:
         # We don't want b and c to be a part of the param enumeration. Just
         # ignore them, because it's not what we want to support in the
         # future.
         assert [str(param.name) for param in m.subscopes[0].params] == result
Beispiel #31
0
def test_append_on_non_sys_path():
    SRC = u("""
class Dummy(object):
    path = []

d = Dummy()
d.path.append('foo')""")
    p = Parser(SRC)
    paths = _check_module(p.module)
    assert len(paths) > 0
    assert 'foo' not in paths
Beispiel #32
0
    def __call__(self, source, module_path=None):
        if not settings.fast_parser:
            return Parser(source, module_path)

        pi = cache.parser_cache.get(module_path, None)
        if pi is None or isinstance(pi.parser, Parser):
            p = super(CachedFastParser, self).__call__(source, module_path)
        else:
            p = pi.parser  # pi is a `cache.ParserCacheItem`
            p.update(source)
        return p
Beispiel #33
0
    def load(buildout_script):
        try:
            with open(buildout_script, 'rb') as f:
                source = common.source_to_unicode(f.read())
        except IOError:
            debug.dbg('Error trying to read buildout_script: %s', buildout_script)
            return

        p = Parser(evaluator.grammar, source, buildout_script)
        cache.save_parser(buildout_script, p)
        return p.module
Beispiel #34
0
        def definition(correct, correct_start, path):
            should_be = set()
            for match in re.finditer('(?:[^ ]+)', correct):
                string = match.group(0)
                parser = Parser(load_grammar(), string, start_symbol='eval_input')
                parser.position_modifier.line = self.line_nr
                element = parser.get_parsed_node()
                element.parent = jedi.api.completion.get_user_scope(
                    script._get_module(),
                    (self.line_nr, self.column)
                )
                results = evaluator.eval_element(element)
                if not results:
                    raise Exception('Could not resolve %s on line %s'
                                    % (match.string, self.line_nr - 1))

                should_be |= set(Definition(evaluator, r) for r in results)

            # Because the objects have different ids, `repr`, then compare.
            should = set(comparison(r) for r in should_be)
            return should
Beispiel #35
0
def _fix_forward_reference(context, node):
    evaled_nodes = context.eval_node(node)
    if len(evaled_nodes) != 1:
        debug.warning("Eval'ed typing index %s should lead to 1 object, "
                      " not %s" % (node, evaled_nodes))
        return node
    evaled_node = list(evaled_nodes)[0]
    if isinstance(evaled_node, compiled.CompiledObject) and \
            isinstance(evaled_node.obj, str):
        try:
            p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj),
                       start_symbol='eval_input')
            new_node = p.get_parsed_node()
        except ParseError:
            debug.warning('Annotation not parsed: %s' % evaled_node.obj)
            return node
        else:
            module = node.get_parent_until()
            new_node.move(module.end_pos[0])
            new_node.parent = context.tree_node
            return new_node
    else:
        return node
Beispiel #36
0
 def test(source, end_pos):
     module = Parser(load_grammar(), u(source)).module
     assert module.get_code() == source
     assert module.end_pos == end_pos
Beispiel #37
0
 def get_call(self, source):
     stmt = Parser(source, no_docstr=True).module.statements[0]
     return stmt.get_commands()[0]
Beispiel #38
0
def test_newline_positions():
    endmarker = Parser(load_grammar(), u('a\n')).module.children[-1]
    assert endmarker.end_pos == (2, 0)
    new_line = endmarker.get_previous()
    assert new_line.start_pos == (1, 1)
    assert new_line.end_pos == (2, 0)
Beispiel #39
0
 def get_call(self, source):
     stmt = Parser(u(source), no_docstr=True).module.statements[0]
     return stmt.expression_list()[0]
Beispiel #40
0
def test_incomplete_list_comprehension():
    """ Shouldn't raise an error, same bug as #418. """
    s = Parser(u('(1 for def')).module.statements[0]
    assert s.expression_list()