Пример #1
0
def _rename(names, replace_str):
    """ For both rename and inline. """
    order = sorted(names, key=lambda x: (x.module_path, x.line, x.column),
                   reverse=True)

    def process(path, old_lines, new_lines):
        if new_lines is not None:  # goto next file, save last
            dct[path] = path, old_lines, new_lines

    dct = {}
    current_path = object()
    new_lines = old_lines = None
    for name in order:
        if name.in_builtin_module():
            continue
        if current_path != name.module_path:
            current_path = name.module_path

            process(current_path, old_lines, new_lines)
            if current_path is not None:
                # None means take the source that is a normal param.
                with open(current_path) as f:
                    source = f.read()

            new_lines = common.splitlines(common.source_to_unicode(source))
            old_lines = new_lines[:]

        nr, indent = name.line, name.column
        line = new_lines[nr - 1]
        new_lines[nr - 1] = line[:indent] + replace_str + \
            line[indent + len(name.name):]
    process(current_path, old_lines, new_lines)
    return dct
Пример #2
0
def extract(script, new_name):
    """ The `args` / `kwargs` params are the same as in `api.Script`.
    :param operation: The refactoring operation to execute.
    :type operation: str
    :type source: str
    :return: list of changed lines/changed files
    """
    new_lines = common.splitlines(common.source_to_unicode(script.source))
    old_lines = new_lines[:]

    user_stmt = script._parser.user_stmt()

    # TODO care for multiline extracts
    dct = {}
    if user_stmt:
        pos = script._pos
        line_index = pos[0] - 1
        arr, index = helpers.array_for_pos(user_stmt, pos)
        if arr is not None:
            start_pos = arr[index].start_pos
            end_pos = arr[index].end_pos

            # take full line if the start line is different from end line
            e = end_pos[1] if end_pos[0] == start_pos[0] else None
            start_line = new_lines[start_pos[0] - 1]
            text = start_line[start_pos[1]:e]
            for l in range(start_pos[0], end_pos[0] - 1):
                text += '\n' + l
            if e is None:
                end_line = new_lines[end_pos[0] - 1]
                text += '\n' + end_line[:end_pos[1]]

            # remove code from new lines
            t = text.lstrip()
            del_start = start_pos[1] + len(text) - len(t)

            text = t.rstrip()
            del_end = len(t) - len(text)
            if e is None:
                new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:]
                e = len(start_line)
            else:
                e = e - del_end
            start_line = start_line[:del_start] + new_name + start_line[e:]
            new_lines[start_pos[0] - 1] = start_line
            new_lines[start_pos[0]:end_pos[0] - 1] = []

            # add parentheses in multiline case
            open_brackets = ['(', '[', '{']
            close_brackets = [')', ']', '}']
            if '\n' in text and not (text[0] in open_brackets and text[-1] ==
                                     close_brackets[open_brackets.index(text[0])]):
                text = '(%s)' % text

            # add new line before statement
            indent = user_stmt.start_pos[1]
            new = "%s%s = %s" % (' ' * indent, new_name, text)
            new_lines.insert(line_index, new)
    dct[script.path] = script.path, old_lines, new_lines
    return Refactoring(dct)
Пример #3
0
def sys_path_with_modifications(evaluator, module):
    if module.path is None:
        # Support for modules without a path is bad, therefore return the
        # normal path.
        return list(get_sys_path())

    curdir = os.path.abspath(os.curdir)
    with common.ignored(OSError):
        os.chdir(os.path.dirname(module.path))

    result = _check_module(evaluator, module)
    result += _detect_django_path(module.path)
    # buildout scripts often contain the same sys.path modifications
    # the set here is used to avoid duplicate sys.path entries
    buildout_paths = set()
    for module_path in _get_buildout_scripts(module.path):
        try:
            with open(module_path, 'rb') as f:
                source = f.read()
        except IOError:
            pass
        else:
            p = Parser(evaluator.grammar, common.source_to_unicode(source), module_path)
            for path in _check_module(p.module):
                if path not in buildout_paths:
                    buildout_paths.add(path)
                    result.append(path)
    # cleanup, back to old directory
    os.chdir(curdir)
    return list(result)
Пример #4
0
    def namespace_packages(self, found_path, import_path):
        """
        Returns a list of paths of possible ``pkgutil``/``pkg_resources``
        namespaces. If the package is no "namespace package", an empty list is
        returned.
        """
        def follow_path(directories, paths):
            try:
                directory = next(directories)
            except StopIteration:
                return paths
            else:
                deeper_paths = []
                for p in paths:
                    new = os.path.join(p, directory)
                    if os.path.isdir(new) and new != found_path:
                        deeper_paths.append(new)
                return follow_path(directories, deeper_paths)

        with open(os.path.join(found_path, '__init__.py'), 'rb') as f:
            content = common.source_to_unicode(f.read())
            # these are strings that need to be used for namespace packages,
            # the first one is ``pkgutil``, the second ``pkg_resources``.
            options = ('declare_namespace(__name__)', 'extend_path(__path__')
            if options[0] in content or options[1] in content:
                # It is a namespace, now try to find the rest of the modules.
                return follow_path((str(i) for i in import_path), sys.path)
        return []
Пример #5
0
 def _py__path__(self):
     search_path = self.evaluator.sys_path
     init_path = self.py__file__()
     if os.path.basename(init_path) == '__init__.py':
         with open(init_path, 'rb') as f:
             content = common.source_to_unicode(f.read())
             # these are strings that need to be used for namespace packages,
             # the first one is ``pkgutil``, the second ``pkg_resources``.
             options = ('declare_namespace(__name__)',
                        'extend_path(__path__')
             if options[0] in content or options[1] in content:
                 # It is a namespace, now try to find the rest of the
                 # modules on sys_path or whatever the search_path is.
                 paths = set()
                 for s in search_path:
                     other = os.path.join(s, self.name.string_name)
                     if os.path.isdir(other):
                         paths.add(other)
                 if paths:
                     return list(paths)
                 # TODO I'm not sure if this is how nested namespace
                 # packages work. The tests are not really good enough to
                 # show that.
     # Default to this.
     return [self._get_init_directory()]
Пример #6
0
    def __init__(self, source=None, line=None, column=None, path=None,
                 encoding='utf-8', source_path=None):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.", DeprecationWarning)
            path = source_path
        self._source_path = path
        self.path = None if path is None else os.path.abspath(path)

        if source is None:
            with open(path) as f:
                source = f.read()

        lines = source.splitlines() or ['']
        if source and source[-1] == '\n':
            lines.append('')
        line = max(len(lines), 1) if line is None else line
        if not (0 < line <= len(lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column

        cache.clear_caches()
        debug.reset_time()
        self.source = common.source_to_unicode(source, encoding)
        self._user_context = UserContext(self.source, self._pos)
        self._parser = UserContextParser(self.source, path, self._pos, self._user_context)
        self._evaluator = Evaluator()
        debug.speed('init')
Пример #7
0
def sys_path_with_modifications(evaluator, module):
    if module.path is None:
        # Support for modules without a path is bad, therefore return the
        # normal path.
        return list(get_sys_path())

    curdir = os.path.abspath(os.curdir)
    with common.ignored(OSError):
        os.chdir(os.path.dirname(module.path))

    result = _check_module(evaluator, module)
    result += _detect_django_path(module.path)
    # buildout scripts often contain the same sys.path modifications
    # the set here is used to avoid duplicate sys.path entries
    buildout_paths = set()
    for module_path in _get_buildout_scripts(module.path):
        try:
            with open(module_path, 'rb') as f:
                source = f.read()
        except IOError:
            pass
        else:
            p = Parser(evaluator.grammar, common.source_to_unicode(source),
                       module_path)
            for path in _check_module(p.module):
                if path not in buildout_paths:
                    buildout_paths.add(path)
                    result.append(path)
    # cleanup, back to old directory
    os.chdir(curdir)
    return list(result)
Пример #8
0
    def namespace_packages(self, found_path, import_path):
        """
        Returns a list of paths of possible ``pkgutil``/``pkg_resources``
        namespaces. If the package is no "namespace package", an empty list is
        returned.
        """
        def follow_path(directories, paths):
            try:
                directory = next(directories)
            except StopIteration:
                return paths
            else:
                deeper_paths = []
                for p in paths:
                    new = os.path.join(p, directory)
                    if os.path.isdir(new) and new != found_path:
                        deeper_paths.append(new)
                return follow_path(directories, deeper_paths)

        with open(os.path.join(found_path, '__init__.py'), 'rb') as f:
            content = common.source_to_unicode(f.read())
            # these are strings that need to be used for namespace packages,
            # the first one is ``pkgutil``, the second ``pkg_resources``.
            options = ('declare_namespace(__name__)', 'extend_path(__path__')
            if options[0] in content or options[1] in content:
                # It is a namespace, now try to find the rest of the modules.
                return follow_path((str(i) for i in import_path), sys.path)
        return []
Пример #9
0
def extract(script, new_name):
    """ The `args` / `kwargs` params are the same as in `api.Script`.
    :param operation: The refactoring operation to execute.
    :type operation: str
    :type source: str
    :return: list of changed lines/changed files
    """
    new_lines = common.source_to_unicode(script.source).splitlines()
    old_lines = new_lines[:]

    user_stmt = script._parser.user_stmt()

    # TODO care for multiline extracts
    dct = {}
    if user_stmt:
        pos = script._pos
        line_index = pos[0] - 1
        arr, index = helpers.array_for_pos(user_stmt, pos)
        if arr is not None:
            start_pos = arr[index].start_pos
            end_pos = arr[index].end_pos

            # take full line if the start line is different from end line
            e = end_pos[1] if end_pos[0] == start_pos[0] else None
            start_line = new_lines[start_pos[0] - 1]
            text = start_line[start_pos[1]:e]
            for l in range(start_pos[0], end_pos[0] - 1):
                text += '\n' + l
            if e is None:
                end_line = new_lines[end_pos[0] - 1]
                text += '\n' + end_line[:end_pos[1]]

            # remove code from new lines
            t = text.lstrip()
            del_start = start_pos[1] + len(text) - len(t)

            text = t.rstrip()
            del_end = len(t) - len(text)
            if e is None:
                new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:]
                e = len(start_line)
            else:
                e = e - del_end
            start_line = start_line[:del_start] + new_name + start_line[e:]
            new_lines[start_pos[0] - 1] = start_line
            new_lines[start_pos[0]:end_pos[0] - 1] = []

            # add parentheses in multiline case
            open_brackets = ['(', '[', '{']
            close_brackets = [')', ']', '}']
            if '\n' in text and not (text[0] in open_brackets and text[-1] ==
                                     close_brackets[open_brackets.index(text[0])]):
                text = '(%s)' % text

            # add new line before statement
            indent = user_stmt.start_pos[1]
            new = "%s%s = %s" % (' ' * indent, new_name, text)
            new_lines.insert(line_index, new)
    dct[script.path] = script.path, old_lines, new_lines
    return Refactoring(dct)
Пример #10
0
def _rename(names, replace_str):
    """ For both rename and inline. """
    order = sorted(names,
                   key=lambda x: (x.module_path, x.line, x.column),
                   reverse=True)

    def process(path, old_lines, new_lines):
        if new_lines is not None:  # goto next file, save last
            dct[path] = path, old_lines, new_lines

    dct = {}
    current_path = object()
    new_lines = old_lines = None
    for name in order:
        if name.in_builtin_module():
            continue
        if current_path != name.module_path:
            current_path = name.module_path

            process(current_path, old_lines, new_lines)
            if current_path is not None:
                # None means take the source that is a normal param.
                with open(current_path) as f:
                    source = f.read()

            new_lines = common.splitlines(common.source_to_unicode(source))
            old_lines = new_lines[:]

        nr, indent = name.line, name.column
        line = new_lines[nr - 1]
        new_lines[nr - 1] = line[:indent] + replace_str + \
            line[indent + len(name.name):]
    process(current_path, old_lines, new_lines)
    return dct
Пример #11
0
 def test_source_to_unicode_unicode_text(self):
     source = (
         b"# vim: fileencoding=utf-8\n"
         b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n"
     )
     actual = common.source_to_unicode(source)
     expected = source.decode('utf-8')
     assert actual == expected
Пример #12
0
 def test_source_to_unicode_unicode_text(self):
     source = (
         b"# vim: fileencoding=utf-8\n"
         b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n"
     )
     actual = common.source_to_unicode(source)
     expected = source.decode('utf-8')
     assert actual == expected
Пример #13
0
 def check_fs(path):
     with open(path, 'rb') as f:
         source = source_to_unicode(f.read())
         if name in source:
             module_name = os.path.basename(path)[:-3]  # Remove `.py`.
             module = _load_module(evaluator, path, source)
             add_module(evaluator, module_name, module)
             return module
Пример #14
0
 def check_fs(path):
     with open(path, 'rb') as f:
         source = source_to_unicode(f.read())
         if name in source:
             module_name = os.path.basename(path)[:-3]  # Remove `.py`.
             module = _load_module(evaluator, path, source)
             add_module(evaluator, module_name, module)
             return module
Пример #15
0
def precache_parser(*files):
    for f in files:
        f = os.path.abspath(f)
        with open(f) as f_:
            source = f_.read()
            source = common.source_to_unicode(source, 'utf-8')
            parser = FastParser(source, f)
            cache.save_parser(f, None, parser, pickling=False)
Пример #16
0
    def load(buildout_script):
        try:
            with open(buildout_script, 'rb') as f:
                source = common.source_to_unicode(f.read())
        except IOError:
            debug.dbg('Error trying to read buildout_script: %s', buildout_script)
            return

        p = Parser(evaluator.grammar, source, buildout_script)
        cache.save_parser(buildout_script, None, p)
        return p.module
Пример #17
0
 def load(source):
     if path is not None and path.endswith('.py'):
         if source is None:
             with open(path) as f:
                 source = f.read()
     else:
         return compiled.load_module(path, name)
     p = path or name
     p = fast.FastParser(common.source_to_unicode(source), p)
     cache.save_parser(path, name, p)
     return p.module
Пример #18
0
    def parent(self):
        obj = self._value
        parser_path = []
        if inspect.ismodule(obj):
            module = obj
        else:
            class FakeParent(pr.Base):
                parent = None  # To avoid having no parent for NamePart.
                path = None

            names = []
            try:
                o = obj.__objclass__
                names.append(obj.__name__)
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                names.insert(0, obj.__name__)
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                module = __import__(module_name)
            fake_name = helpers.FakeName(names, FakeParent())
            parser_path = fake_name.names
        raw_module = get_module(self._value)

        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            path = re.sub('c$', '', path)
            if path.endswith('.py'):
                # cut the `c` from `.pyc`
                with open(path) as f:
                    source = source_to_unicode(f.read())
                mod = FastParser(source, path[:-1]).module
                if not parser_path:
                    return mod
                found = self._evaluator.eval_call_path(iter(parser_path), mod, None)
                if found:
                    return found[0]
                debug.warning('Interpreter lookup for Python code failed %s',
                              mod)

        module = compiled.CompiledObject(raw_module)
        if raw_module == builtins:
            # The builtins module is special and always cached.
            module = compiled.builtin
        return compiled.create(self._evaluator, self._value, module, module)
Пример #19
0
 def load(source):
     if path is not None and path.endswith('.py'):
         if source is None:
             with open(path) as f:
                 source = f.read()
     else:
         return compiled.load_module(path, name)
     p = path or name
     p = fast.FastParser(common.source_to_unicode(source), p)
     cache.save_parser(path, name, p)
     return p.module
Пример #20
0
    def load(buildout_script):
        try:
            with open(buildout_script, "rb") as f:
                source = common.source_to_unicode(f.read())
        except IOError:
            debug.dbg("Error trying to read buildout_script: %s", buildout_script)
            return

        p = ParserWithRecovery(evaluator.grammar, source, buildout_script)
        save_parser(buildout_script, p)
        return p.module
Пример #21
0
    def load(buildout_script):
        try:
            with open(buildout_script, 'rb') as f:
                source = common.source_to_unicode(f.read())
        except IOError:
            debug.dbg('Error trying to read buildout_script: %s', buildout_script)
            return

        p = Parser(evaluator.grammar, source, buildout_script)
        cache.save_parser(buildout_script, p)
        return p.module
Пример #22
0
    def __init__(self,
                 source=None,
                 line=None,
                 column=None,
                 path=None,
                 encoding='utf-8',
                 source_path=None,
                 source_encoding=None,
                 sys_path=None):
        if source_path is not None:
            warnings.warn(
                "Deprecated since version 0.7. Use path instead of source_path.",
                DeprecationWarning,
                stacklevel=2)
            path = source_path
        if source_encoding is not None:
            warnings.warn(
                "Deprecated since version 0.8. Use encoding instead of source_encoding.",
                DeprecationWarning,
                stacklevel=2)
            encoding = source_encoding

        self._orig_path = path
        # An empty path (also empty string) should always result in no path.
        self.path = os.path.abspath(path) if path else None

        if source is None:
            # TODO add a better warning than the traceback!
            with open(path, 'rb') as f:
                source = f.read()

        self._source = common.source_to_unicode(source, encoding)
        self._code_lines = common.splitlines(self._source)
        line = max(len(self._code_lines), 1) if line is None else line
        if not (0 < line <= len(self._code_lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(self._code_lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column
        self._path = path

        cache.clear_time_caches()
        debug.reset_time()
        self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
        if sys_path is None:
            venv = os.getenv('VIRTUAL_ENV')
            if venv:
                sys_path = list(get_venv_path(venv))
        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
        debug.speed('init')
Пример #23
0
    def __init__(self,
                 source=None,
                 line=None,
                 column=None,
                 path=None,
                 encoding='utf-8',
                 source_path=None,
                 source_encoding=None,
                 sys_path=None):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.",
                          DeprecationWarning)
            path = source_path
        if source_encoding is not None:
            warnings.warn("Use encoding instead of source_encoding.",
                          DeprecationWarning)
            encoding = source_encoding

        self._orig_path = path
        # An empty path (also empty string) should always result in no path.
        self.path = os.path.abspath(path) if path else None

        if source is None:
            # TODO add a better warning than the traceback!
            try:
                with open(path) as f:
                    source = f.read()
            except UnicodeDecodeError:
                with open(path, encoding=encoding) as f:
                    source = f.read()

        self._source = common.source_to_unicode(source, encoding)
        self._code_lines = common.splitlines(self._source)
        line = max(len(self._code_lines), 1) if line is None else line
        if not (0 < line <= len(self._code_lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(self._code_lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column
        self._path = path

        cache.clear_time_caches()
        debug.reset_time()
        self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2])
        if sys_path is None:
            venv = os.getenv('VIRTUAL_ENV')
            if venv:
                sys_path = list(get_venv_path(venv))
        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
        debug.speed('init')
Пример #24
0
 def load(source):
     dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
     if path is not None and path.endswith(".py") and not dotted_path in settings.auto_import_modules:
         if source is None:
             with open(path, "rb") as f:
                 source = f.read()
     else:
         return compiled.load_module(path)
     p = path
     p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
     cache.save_parser(path, p)
     return p.module
Пример #25
0
 def load(source):
     dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
     if path is not None and path.endswith('.py') \
             and not dotted_path in settings.auto_import_modules:
         if source is None:
             with open(path, 'rb') as f:
                 source = f.read()
     else:
         return compiled.load_module(path, name)
     p = path or name
     p = fast.FastParser(common.source_to_unicode(source), p)
     cache.save_parser(path, name, p)
     return p.module
Пример #26
0
 def load(source):
     dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
     if path is not None and path.endswith(('.py', '.zip', '.egg')) \
             and dotted_path not in settings.auto_import_modules:
         if source is None:
             with open(path, 'rb') as f:
                 source = f.read()
     else:
         return compiled.load_module(evaluator, path)
     p = path
     p = FastParser(evaluator.grammar, source_to_unicode(source), p)
     save_parser(path, p)
     return p.module
Пример #27
0
 def load(source):
     dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
     if path is not None and path.endswith('.py') \
             and not dotted_path in settings.auto_import_modules:
         if source is None:
             with open(path, 'rb') as f:
                 source = f.read()
     else:
         return compiled.load_module(path, name)
     p = path or name
     p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
     cache.save_parser(path, name, p)
     return p.module
Пример #28
0
    def __init__(self,
                 source=None,
                 line=None,
                 column=None,
                 path=None,
                 encoding='utf-8',
                 source_path=None,
                 source_encoding=None,
                 sys_path=None):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.",
                          DeprecationWarning)
            path = source_path
        if source_encoding is not None:
            warnings.warn("Use encoding instead of source_encoding.",
                          DeprecationWarning)
            encoding = source_encoding

        self._orig_path = path
        self.path = None if path is None else os.path.abspath(path)

        if source is None:
            with open(path) as f:
                source = f.read()

        self.source = common.source_to_unicode(source, encoding)
        lines = common.splitlines(self.source)
        line = max(len(lines), 1) if line is None else line
        if not (0 < line <= len(lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column

        cache.clear_time_caches()
        debug.reset_time()
        self._grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
        self._user_context = UserContext(self.source, self._pos)
        self._parser = UserContextParser(self._grammar, self.source, path,
                                         self._pos, self._user_context,
                                         self._parsed_callback)
        if sys_path is None:
            venv = os.getenv('VIRTUAL_ENV')
            if venv:
                sys_path = list(get_venv_path(venv))
        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
        debug.speed('init')
 def load(source):
     dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
     if path is not None and path.endswith(('.py', '.zip', '.egg')) \
             and dotted_path not in settings.auto_import_modules:
         if source is None:
             with open(path, 'rb') as f:
                 source = f.read()
     else:
         return compiled.load_module(evaluator, path)
     p = path
     p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p)
     save_parser(path, p)
     from jedi.evaluate.representation import ModuleWrapper
     return ModuleWrapper(evaluator, p.module, parent_module)
Пример #30
0
 def load(source):
     dotted_path = path and compiled.dotted_from_fs_path(path, sys_path)
     if path is not None and path.endswith(('.py', '.zip', '.egg')) \
             and dotted_path not in settings.auto_import_modules:
         if source is None:
             with open(path, 'rb') as f:
                 source = f.read()
     else:
         return compiled.load_module(evaluator, path)
     p = path
     p = fast.FastParser(evaluator.grammar,
                         common.source_to_unicode(source), p)
     save_parser(path, p)
     from jedi.evaluate.representation import ModuleWrapper
     return ModuleWrapper(evaluator, p.module, parent_module)
Пример #31
0
def inline(script):
    """
    :type script: api.Script
    """
    new_lines = common.splitlines(common.source_to_unicode(script.source))

    dct = {}

    definitions = script.goto_assignments()
    with common.ignored(AssertionError):
        assert len(definitions) == 1
        stmt = definitions[0]._definition
        usages = script.usages()
        inlines = [
            r for r in usages
            if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos
        ]
        inlines = sorted(inlines,
                         key=lambda x: (x.module_path, x.line, x.column),
                         reverse=True)
        expression_list = stmt.expression_list()
        # don't allow multiline refactorings for now.
        assert stmt.start_pos[0] == stmt.end_pos[0]
        index = stmt.start_pos[0] - 1

        line = new_lines[index]
        replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
        replace_str = replace_str.strip()
        # tuples need parentheses
        if expression_list and isinstance(expression_list[0], pr.Array):
            arr = expression_list[0]
            if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
                replace_str = '(%s)' % replace_str

        # if it's the only assignment, remove the statement
        if len(stmt.get_defined_names()) == 1:
            line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]

        dct = _rename(inlines, replace_str)
        # remove the empty line
        new_lines = dct[script.path][2]
        if line.strip():
            new_lines[index] = line
        else:
            new_lines.pop(index)

    return Refactoring(dct)
Пример #32
0
    def parent(self):
        parser_path = []
        obj = self._value
        if inspect.ismodule(obj):
            module = obj
        else:
            try:
                o = obj.__objclass__
                parser_path.append(
                    pr.NamePart(obj.__name__, None, (None, None)))
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                parser_path.insert(
                    0, pr.NamePart(obj.__name__, None, (None, None)))
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                module = __import__(module_name)
        raw_module = get_module(self._value)

        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            path = re.sub('c$', '', path)
            if path.endswith('.py'):
                # cut the `c` from `.pyc`
                with open(path) as f:
                    source = source_to_unicode(f.read())
                mod = FastParser(source, path[:-1]).module
                if not parser_path:
                    return mod
                found = self._evaluator.eval_call_path(iter(parser_path), mod,
                                                       None)
                if found:
                    return found[0]
                debug.warning('Interpreter lookup for Python code failed %s',
                              mod)

        module = compiled.CompiledObject(raw_module)
        return compiled.create(self._value, module, module)
Пример #33
0
def defined_names(source, path=None, encoding='utf-8'):
    """
    Get all definitions in `source` sorted by its position.

    This functions can be used for listing functions, classes and
    data defined in a file.  This can be useful if you want to list
    them in "sidebar".  Each element in the returned list also has
    `defined_names` method which can be used to get sub-definitions
    (e.g., methods in class).

    :rtype: list of classes.Definition
    """
    parser = Parser(
        common.source_to_unicode(source, encoding),
        module_path=path,
    )
    return classes.defined_names(Evaluator(), parser.module)
Пример #34
0
def defined_names(source, path=None, encoding='utf-8'):
    """
    Get all definitions in `source` sorted by its position.

    This functions can be used for listing functions, classes and
    data defined in a file.  This can be useful if you want to list
    them in "sidebar".  Each element in the returned list also has
    `defined_names` method which can be used to get sub-definitions
    (e.g., methods in class).

    :rtype: list of classes.Definition
    """
    parser = Parser(
        common.source_to_unicode(source, encoding),
        module_path=path,
    )
    return classes.defined_names(Evaluator(), parser.module)
Пример #35
0
    def __init__(self,
                 source=None,
                 line=None,
                 column=None,
                 path=None,
                 encoding='utf-8',
                 source_path=None,
                 source_encoding=None):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.",
                          DeprecationWarning)
            path = source_path
        if source_encoding is not None:
            warnings.warn("Use encoding instead of source_encoding.",
                          DeprecationWarning)
            encoding = source_encoding

        self._orig_path = path
        self.path = None if path is None else os.path.abspath(path)

        if source is None:
            with open(path) as f:
                source = f.read()

        lines = source.splitlines() or ['']
        if source and source[-1] == '\n':
            lines.append('')
        line = max(len(lines), 1) if line is None else line
        if not (0 < line <= len(lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column

        cache.clear_caches()
        debug.reset_time()
        self.source = common.source_to_unicode(source, encoding)
        self._user_context = UserContext(self.source, self._pos)
        self._parser = UserContextParser(self.source, path, self._pos,
                                         self._user_context)
        self._evaluator = Evaluator()
        debug.speed('init')
Пример #36
0
    def parent(self):
        parser_path = []
        obj = self._value
        if inspect.ismodule(obj):
            module = obj
        else:
            try:
                o = obj.__objclass__
                parser_path.append(pr.NamePart(obj.__name__, None, (None, None)))
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                parser_path.insert(0, pr.NamePart(obj.__name__, None, (None, None)))
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                module = __import__(module_name)
        raw_module = get_module(self._value)

        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            path = re.sub('c$', '', path)
            if path.endswith('.py'):
                # cut the `c` from `.pyc`
                with open(path) as f:
                    source = source_to_unicode(f.read())
                mod = FastParser(source, path[:-1]).module
                if not parser_path:
                    return mod
                found = self._evaluator.eval_call_path(iter(parser_path), mod, None)
                if found:
                    return found[0]
                debug.warning('Interpreter lookup for Python code failed %s',
                              mod)

        module = compiled.CompiledObject(raw_module)
        return compiled.create(self._value, module, module)
Пример #37
0
def inline(script):
    """
    :type script: api.Script
    """
    new_lines = common.splitlines(common.source_to_unicode(script.source))

    dct = {}

    definitions = script.goto_assignments()
    with common.ignored(AssertionError):
        assert len(definitions) == 1
        stmt = definitions[0]._definition
        usages = script.usages()
        inlines = [r for r in usages
                   if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos]
        inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column),
                         reverse=True)
        expression_list = stmt.expression_list()
        # don't allow multiline refactorings for now.
        assert stmt.start_pos[0] == stmt.end_pos[0]
        index = stmt.start_pos[0] - 1

        line = new_lines[index]
        replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1]
        replace_str = replace_str.strip()
        # tuples need parentheses
        if expression_list and isinstance(expression_list[0], pr.Array):
            arr = expression_list[0]
            if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1:
                replace_str = '(%s)' % replace_str

        # if it's the only assignment, remove the statement
        if len(stmt.get_defined_names()) == 1:
            line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:]

        dct = _rename(inlines, replace_str)
        # remove the empty line
        new_lines = dct[script.path][2]
        if line.strip():
            new_lines[index] = line
        else:
            new_lines.pop(index)

    return Refactoring(dct)
Пример #38
0
    def __init__(self, source=None, line=None, column=None, path=None,
                 encoding='utf-8', source_path=None, source_encoding=None,
                 sys_path=None):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.", DeprecationWarning)
            path = source_path
        if source_encoding is not None:
            warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning)
            encoding = source_encoding

        self._orig_path = path
        self.path = None if path is None else os.path.abspath(path)

        if source is None:
            with open(path) as f:
                source = f.read()

        self.source = common.source_to_unicode(source, encoding)
        lines = common.splitlines(self.source)
        line = max(len(lines), 1) if line is None else line
        if not (0 < line <= len(lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column

        cache.clear_time_caches()
        debug.reset_time()
        self._grammar = load_grammar('grammar%s.%s' % sys.version_info[:2])
        self._user_context = UserContext(self.source, self._pos)
        self._parser = UserContextParser(self._grammar, self.source, path,
                                         self._pos, self._user_context,
                                         self._parsed_callback)
        if sys_path is None:
            venv = os.getenv('VIRTUAL_ENV')
            if venv:
                sys_path = list(get_venv_path(venv))
        self._evaluator = Evaluator(self._grammar, sys_path=sys_path)
        debug.speed('init')
Пример #39
0
        def return_value(search_path):
            init_path = self.py__file__()
            if os.path.basename(init_path) == '__init__.py':

                with open(init_path, 'rb') as f:
                    content = common.source_to_unicode(f.read())
                    # these are strings that need to be used for namespace packages,
                    # the first one is ``pkgutil``, the second ``pkg_resources``.
                    options = ('declare_namespace(__name__)', 'extend_path(__path__')
                    if options[0] in content or options[1] in content:
                        # It is a namespace, now try to find the rest of the
                        # modules on sys_path or whatever the search_path is.
                        paths = set()
                        for s in search_path:
                            other = os.path.join(s, unicode(self.name))
                            if os.path.isdir(other):
                                paths.add(other)
                        return list(paths)
            # Default to this.
            return [path]
Пример #40
0
        def return_value(search_path):
            init_path = self.py__file__()
            if os.path.basename(init_path) == '__init__.py':

                with open(init_path, 'rb') as f:
                    content = common.source_to_unicode(f.read())
                    # these are strings that need to be used for namespace packages,
                    # the first one is ``pkgutil``, the second ``pkg_resources``.
                    options = ('declare_namespace(__name__)',
                               'extend_path(__path__')
                    if options[0] in content or options[1] in content:
                        # It is a namespace, now try to find the rest of the
                        # modules on sys_path or whatever the search_path is.
                        paths = set()
                        for s in search_path:
                            other = os.path.join(s, unicode(self.name))
                            if os.path.isdir(other):
                                paths.add(other)
                        return list(paths)
            # Default to this.
            return [path]
Пример #41
0
    def __init__(self, source=None, line=None, column=None, path=None,
                 encoding='utf-8', source_path=None, source_encoding=None,
                 resolve_variables_to_types=True):
        if source_path is not None:
            warnings.warn("Use path instead of source_path.", DeprecationWarning)
            path = source_path
        if source_encoding is not None:
            warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning)
            encoding = source_encoding

        self._orig_path = path
        self.path = None if path is None else os.path.abspath(path)

        if source is None:
            with open(path) as f:
                source = f.read()

        self.source = common.source_to_unicode(source, encoding)
        lines = common.splitlines(self.source)
        line = max(len(lines), 1) if line is None else line
        if not (0 < line <= len(lines)):
            raise ValueError('`line` parameter is not in a valid range.')

        line_len = len(lines[line - 1])
        column = line_len if column is None else column
        if not (0 <= column <= line_len):
            raise ValueError('`column` parameter is not in a valid range.')
        self._pos = line, column

        if not cache.never_clear_cache:
            cache.clear_caches()

        debug.reset_time()
        self._user_context = UserContext(self.source, self._pos)
        self._parser = UserContextParser(self.source, path, self._pos, self._user_context)
        self._evaluator = Evaluator(resolve_variables_to_types=resolve_variables_to_types)
        debug.speed('init')
Пример #42
0
 def _py__path__(self):
     search_path = self.evaluator.sys_path
     init_path = self.py__file__()
     if os.path.basename(init_path) == '__init__.py':
         with open(init_path, 'rb') as f:
             content = common.source_to_unicode(f.read())
             # these are strings that need to be used for namespace packages,
             # the first one is ``pkgutil``, the second ``pkg_resources``.
             options = ('declare_namespace(__name__)', 'extend_path(__path__')
             if options[0] in content or options[1] in content:
                 # It is a namespace, now try to find the rest of the
                 # modules on sys_path or whatever the search_path is.
                 paths = set()
                 for s in search_path:
                     other = os.path.join(s, self.name.string_name)
                     if os.path.isdir(other):
                         paths.add(other)
                 if paths:
                     return list(paths)
                 # TODO I'm not sure if this is how nested namespace
                 # packages work. The tests are not really good enough to
                 # show that.
     # Default to this.
     return [self._get_init_directory()]
Пример #43
0
def parse(grammar, path):
    with open(path) as f:
        source = f.read()
    source = common.source_to_unicode(source)
    return FastParser(grammar, source, path)
Пример #44
0
 def check_fs(path):
     with open(path) as f:
         source = source_to_unicode(f.read())
         if name in source:
             return load_module(path, source)
Пример #45
0
    def parent(self):
        """
        Creating fake statements for the interpreter.

        Here we are trying to link back to Python code, if possible. This means
        we try to find the python module for a name (not the builtin).
        """
        return mixed.create(self._evaluator, self._value)
        obj = self._value
        parser_path = []
        if inspect.ismodule(obj):
            module = obj
        else:
            names = []
            try:
                o = obj.__objclass__
                names.append(obj.__name__)
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                names.insert(0, obj.__name__)
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                # If we put anything into fromlist, it will just return the
                # latest name.
                module = __import__(module_name, fromlist=[''])
            parser_path = names

        found = []
        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            # Find the corresponding Python file for an interpreted one.
            path = re.sub(r'\.pyc$', '.py', path)

            if path.endswith('.py'):
                with open(path) as f:
                    source = source_to_unicode(f.read())
                mod = FastParser(load_grammar(), source, path).module
                mod = self._evaluator.wrap(mod)

                # We have to make sure that the modules that we import are also
                # part of evaluator.modules.
                for module_name, module in sys.modules.items():
                    try:
                        iterated_path = module.__file__
                    except AttributeError:
                        pass
                    else:
                        if iterated_path == path:
                            self._evaluator.modules[module_name] = mod
                            break
                else:
                    raise NotImplementedError('This should not happen, a module '
                                              'should be part of sys.modules.')

                if parser_path:
                    assert len(parser_path) == 1
                    found = list(self._evaluator.find_types(mod, parser_path[0],
                                                            search_global=True))
                else:
                    found = [mod]

                if not found:
                    debug.warning('Interpreter lookup failed in global scope for %s',
                                  parser_path)

        if not found:
            evaluated = compiled.create(self._evaluator, obj)
            found = [evaluated]

        if len(found) > 1:
            content = iterable.AlreadyEvaluated(found)
            stmt = pt.ExprStmt([self, pt.Operator(pt.zero_position_modifier,
                                                  '=', (0, 0), ''), content])
            stmt.parent = self._module
            return stmt
        else:
            return found[0]
Пример #46
0
def parse(code=None, path=None, grammar=None, error_recovery=True,
          start_symbol='file_input', cache=False, diff_cache=False):
    """
    If you want to parse a Python file you want to start here, most likely.

    If you need finer grained control over the parsed instance, there will be
    other ways to access it.

    :param code: A unicode string that contains Python code.
    :param path: The path to the file you want to open. Only needed for caching.
    :param grammar: A Python grammar file, created with load_grammar. You may
        not specify it. In that case it's the current Python version.
    :param error_recovery: If enabled, any code will be returned. If it is
        invalid, it will be returned as an error node. If disabled, you will
        get a ParseError when encountering syntax errors in your code.
    :param start_symbol: The grammar symbol that you want to parse. Only
        allowed to be used when error_recovery is disabled.

    :return: A syntax tree node. Typically the module.
    """
    if code is None and path is None:
        raise TypeError("Please provide either code or a path.")

    if grammar is None:
        grammar = load_grammar()

    if cache and not code and path is not None:
        # In this case we do actual caching. We just try to load it.
        module_node = load_module(grammar, path)
        if module_node is not None:
            return module_node

    if code is None:
        with open(path, 'rb') as f:
            code = source_to_unicode(f.read())

    if diff_cache and settings.fast_parser:
        try:
            module_cache_item = parser_cache[path]
        except KeyError:
            pass
        else:
            lines = splitlines(code, keepends=True)
            module_node = module_cache_item.node
            old_lines = module_cache_item.lines
            if old_lines == lines:
                save_module(grammar, path, module_node, lines, pickling=False)
                return module_node

            new_node = DiffParser(grammar, module_node).update(
                old_lines=old_lines,
                new_lines=lines
            )
            save_module(grammar, path, new_node, lines, pickling=cache)
            return new_node

    added_newline = not code.endswith('\n')
    lines = tokenize_lines = splitlines(code, keepends=True)
    if added_newline:
        code += '\n'
        tokenize_lines = list(tokenize_lines)
        tokenize_lines[-1] += '\n'
        tokenize_lines.append('')

    tokens = generate_tokens(tokenize_lines, use_exact_op_types=True)

    p = Parser(grammar, error_recovery=error_recovery, start_symbol=start_symbol)
    root_node = p.parse(tokens=tokens)
    if added_newline:
        _remove_last_newline(root_node)

    if cache or diff_cache:
        save_module(grammar, path, root_node, lines, pickling=cache)
    return root_node
Пример #47
0
    def parent(self):
        """
        Creating fake statements for the interpreter.
        """
        obj = self._value
        parser_path = []
        if inspect.ismodule(obj):
            module = obj
        else:
            names = []
            try:
                o = obj.__objclass__
                names.append(obj.__name__)
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                names.insert(0, obj.__name__)
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                # TODO this import is wrong. Yields x for x.y.z instead of z
                module = __import__(module_name)
            parser_path = names
        raw_module = get_module(self._value)

        found = []
        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            path = re.sub('c$', '', path)
            if path.endswith('.py'):
                # cut the `c` from `.pyc`
                with open(path) as f:
                    source = source_to_unicode(f.read())
                mod = FastParser(load_grammar(), source, path[:-1]).module
                if parser_path:
                    assert len(parser_path) == 1
                    found = self._evaluator.find_types(mod,
                                                       parser_path[0],
                                                       search_global=True)
                else:
                    found = [er.wrap(self._evaluator, mod)]

                if not found:
                    debug.warning(
                        'Possibly an interpreter lookup for Python code failed %s',
                        parser_path)

        if not found:
            evaluated = compiled.CompiledObject(obj)
            if evaluated == builtins:
                # The builtins module is special and always cached.
                evaluated = compiled.builtin
            found = [evaluated]

        content = iterable.AlreadyEvaluated(found)
        stmt = pt.ExprStmt([
            self,
            pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content
        ])
        stmt.parent = self._module
        return stmt
Пример #48
0
    def parent(self):
        """
        Creating fake statements for the interpreter.

        Here we are trying to link back to Python code, if possible. This means
        we try to find the python module for a name (not the builtin).
        """
        return mixed.create(self._evaluator, self._value)
        obj = self._value
        parser_path = []
        if inspect.ismodule(obj):
            module = obj
        else:
            names = []
            try:
                o = obj.__objclass__
                names.append(obj.__name__)
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                names.insert(0, obj.__name__)
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                # If we put anything into fromlist, it will just return the
                # latest name.
                module = __import__(module_name, fromlist=[''])
            parser_path = names

        found = []
        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            # Find the corresponding Python file for an interpreted one.
            path = re.sub(r'\.pyc$', '.py', path)

            if path.endswith('.py'):
                with open(path) as f:
                    source = source_to_unicode(f.read())
                mod = FastParser(load_grammar(), source, path).module
                mod = self._evaluator.wrap(mod)

                # We have to make sure that the modules that we import are also
                # part of evaluator.modules.
                for module_name, module in sys.modules.items():
                    try:
                        iterated_path = module.__file__
                    except AttributeError:
                        pass
                    else:
                        if iterated_path == path:
                            self._evaluator.modules[module_name] = mod
                            break
                else:
                    raise NotImplementedError(
                        'This should not happen, a module '
                        'should be part of sys.modules.')

                if parser_path:
                    assert len(parser_path) == 1
                    found = list(
                        self._evaluator.find_types(mod,
                                                   parser_path[0],
                                                   search_global=True))
                else:
                    found = [mod]

                if not found:
                    debug.warning(
                        'Interpreter lookup failed in global scope for %s',
                        parser_path)

        if not found:
            evaluated = compiled.create(self._evaluator, obj)
            found = [evaluated]

        if len(found) > 1:
            content = iterable.AlreadyEvaluated(found)
            stmt = pt.ExprStmt([
                self,
                pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''),
                content
            ])
            stmt.parent = self._module
            return stmt
        else:
            return found[0]
Пример #49
0
 def check_fs(path):
     with open(path, 'rb') as f:
         source = source_to_unicode(f.read())
         if name in source:
             return _load_module(evaluator, path, source)
Пример #50
0
 def check_fs(path):
     with open(path, 'rb') as f:
         source = source_to_unicode(f.read())
         if name in source:
             return _load_module(evaluator, path, source)
Пример #51
0
def parse(grammar, path):
    with open(path) as f:
        source = f.read()
    source = common.source_to_unicode(source)
    return FastParser(grammar, source, path)
Пример #52
0
    def parent(self):
        """
        Creating fake statements for the interpreter.
        """
        obj = self._value
        parser_path = []
        if inspect.ismodule(obj):
            module = obj
        else:
            names = []
            try:
                o = obj.__objclass__
                names.append(obj.__name__)
                obj = o
            except AttributeError:
                pass

            try:
                module_name = obj.__module__
                names.insert(0, obj.__name__)
            except AttributeError:
                # Unfortunately in some cases like `int` there's no __module__
                module = builtins
            else:
                # TODO this import is wrong. Yields x for x.y.z instead of z
                module = __import__(module_name)
            parser_path = names
        raw_module = get_module(self._value)

        found = []
        try:
            path = module.__file__
        except AttributeError:
            pass
        else:
            path = re.sub('c$', '', path)
            if path.endswith('.py'):
                # cut the `c` from `.pyc`
                with open(path) as f:
                    source = source_to_unicode(f.read())
                mod = FastParser(load_grammar(), source, path[:-1]).module
                if parser_path:
                    assert len(parser_path) == 1
                    found = self._evaluator.find_types(mod, parser_path[0], search_global=True)
                else:
                    found = [er.wrap(self._evaluator, mod)]

                if not found:
                    debug.warning('Possibly an interpreter lookup for Python code failed %s',
                                  parser_path)

        if not found:
            evaluated = compiled.CompiledObject(obj)
            if evaluated == builtins:
                # The builtins module is special and always cached.
                evaluated = compiled.builtin
            found = [evaluated]

        content = iterable.AlreadyEvaluated(found)
        stmt = pt.ExprStmt([self, pt.Operator(pt.zero_position_modifier,
                                              '=', (0, 0), ''), content])
        stmt.parent = self._module
        return stmt
Пример #53
0
 def check_fs(path):
     with open(path) as f:
         source = source_to_unicode(f.read())
         if name in source:
             return load_module(path, source)
Пример #54
0
def parse(code=None, path=None, grammar=None, error_recovery=True,
          start_symbol='file_input', cache=False, diff_cache=False):
    """
    If you want to parse a Python file you want to start here, most likely.

    If you need finer grained control over the parsed instance, there will be
    other ways to access it.

    :param code: A unicode string that contains Python code.
    :param path: The path to the file you want to open. Only needed for caching.
    :param grammar: A Python grammar file, created with load_grammar.
    :param error_recovery: If enabled, any code will be returned. If it is
        invalid, it will be returned as an error node. If disabled, you will
        get a ParseError when encountering syntax errors in your code.
    :param start_symbol: The grammar symbol that you want to parse. Only
        allowed to be used when error_recovery is disabled.

    :return: A syntax tree node. Typically the module.
    """
    if code is None and path is None:
        raise TypeError("Please provide either code or a path.")

    if grammar is None:
        grammar = load_grammar()

    if path is not None:
        path = os.path.expanduser(path)

    if cache and not code and path is not None:
        # In this case we do actual caching. We just try to load it.
        module_node = load_module(grammar, path)
        if module_node is not None:
            return module_node

    if code is None:
        with open(path, 'rb') as f:
            code = source_to_unicode(f.read())

    if diff_cache and settings.fast_parser:
        try:
            module_cache_item = parser_cache[path]
        except KeyError:
            pass
        else:
            lines = splitlines(code, keepends=True)
            module_node = module_cache_item.node
            old_lines = module_cache_item.lines
            if old_lines == lines:
                save_module(grammar, path, module_node, lines, pickling=False)
                return module_node

            new_node = DiffParser(grammar, module_node).update(
                old_lines=old_lines,
                new_lines=lines
            )
            save_module(grammar, path, new_node, lines, pickling=cache)
            return new_node

    added_newline = not code.endswith('\n')
    lines = tokenize_lines = splitlines(code, keepends=True)
    if added_newline:
        code += '\n'
        tokenize_lines = list(tokenize_lines)
        tokenize_lines[-1] += '\n'
        tokenize_lines.append('')

    tokens = generate_tokens(tokenize_lines, use_exact_op_types=True)

    p = Parser(grammar, error_recovery=error_recovery, start_symbol=start_symbol)
    root_node = p.parse(tokens=tokens)
    if added_newline:
        _remove_last_newline(root_node)

    if cache or diff_cache:
        save_module(grammar, path, root_node, lines, pickling=cache)
    return root_node