def _rename(names, replace_str): """ For both rename and inline. """ order = sorted(names, key=lambda x: (x.module_path, x.line, x.column), reverse=True) def process(path, old_lines, new_lines): if new_lines is not None: # goto next file, save last dct[path] = path, old_lines, new_lines dct = {} current_path = object() new_lines = old_lines = None for name in order: if name.in_builtin_module(): continue if current_path != name.module_path: current_path = name.module_path process(current_path, old_lines, new_lines) if current_path is not None: # None means take the source that is a normal param. with open(current_path) as f: source = f.read() new_lines = common.splitlines(common.source_to_unicode(source)) old_lines = new_lines[:] nr, indent = name.line, name.column line = new_lines[nr - 1] new_lines[nr - 1] = line[:indent] + replace_str + \ line[indent + len(name.name):] process(current_path, old_lines, new_lines) return dct
def extract(script, new_name): """ The `args` / `kwargs` params are the same as in `api.Script`. :param operation: The refactoring operation to execute. :type operation: str :type source: str :return: list of changed lines/changed files """ new_lines = common.splitlines(common.source_to_unicode(script.source)) old_lines = new_lines[:] user_stmt = script._parser.user_stmt() # TODO care for multiline extracts dct = {} if user_stmt: pos = script._pos line_index = pos[0] - 1 arr, index = helpers.array_for_pos(user_stmt, pos) if arr is not None: start_pos = arr[index].start_pos end_pos = arr[index].end_pos # take full line if the start line is different from end line e = end_pos[1] if end_pos[0] == start_pos[0] else None start_line = new_lines[start_pos[0] - 1] text = start_line[start_pos[1]:e] for l in range(start_pos[0], end_pos[0] - 1): text += '\n' + l if e is None: end_line = new_lines[end_pos[0] - 1] text += '\n' + end_line[:end_pos[1]] # remove code from new lines t = text.lstrip() del_start = start_pos[1] + len(text) - len(t) text = t.rstrip() del_end = len(t) - len(text) if e is None: new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:] e = len(start_line) else: e = e - del_end start_line = start_line[:del_start] + new_name + start_line[e:] new_lines[start_pos[0] - 1] = start_line new_lines[start_pos[0]:end_pos[0] - 1] = [] # add parentheses in multiline case open_brackets = ['(', '[', '{'] close_brackets = [')', ']', '}'] if '\n' in text and not (text[0] in open_brackets and text[-1] == close_brackets[open_brackets.index(text[0])]): text = '(%s)' % text # add new line before statement indent = user_stmt.start_pos[1] new = "%s%s = %s" % (' ' * indent, new_name, text) new_lines.insert(line_index, new) dct[script.path] = script.path, old_lines, new_lines return Refactoring(dct)
def sys_path_with_modifications(evaluator, module): if module.path is None: # Support for modules without a path is bad, therefore return the # normal path. return list(get_sys_path()) curdir = os.path.abspath(os.curdir) with common.ignored(OSError): os.chdir(os.path.dirname(module.path)) result = _check_module(evaluator, module) result += _detect_django_path(module.path) # buildout scripts often contain the same sys.path modifications # the set here is used to avoid duplicate sys.path entries buildout_paths = set() for module_path in _get_buildout_scripts(module.path): try: with open(module_path, 'rb') as f: source = f.read() except IOError: pass else: p = Parser(evaluator.grammar, common.source_to_unicode(source), module_path) for path in _check_module(p.module): if path not in buildout_paths: buildout_paths.add(path) result.append(path) # cleanup, back to old directory os.chdir(curdir) return list(result)
def namespace_packages(self, found_path, import_path): """ Returns a list of paths of possible ``pkgutil``/``pkg_resources`` namespaces. If the package is no "namespace package", an empty list is returned. """ def follow_path(directories, paths): try: directory = next(directories) except StopIteration: return paths else: deeper_paths = [] for p in paths: new = os.path.join(p, directory) if os.path.isdir(new) and new != found_path: deeper_paths.append(new) return follow_path(directories, deeper_paths) with open(os.path.join(found_path, '__init__.py'), 'rb') as f: content = common.source_to_unicode(f.read()) # these are strings that need to be used for namespace packages, # the first one is ``pkgutil``, the second ``pkg_resources``. options = ('declare_namespace(__name__)', 'extend_path(__path__') if options[0] in content or options[1] in content: # It is a namespace, now try to find the rest of the modules. return follow_path((str(i) for i in import_path), sys.path) return []
def _py__path__(self): search_path = self.evaluator.sys_path init_path = self.py__file__() if os.path.basename(init_path) == '__init__.py': with open(init_path, 'rb') as f: content = common.source_to_unicode(f.read()) # these are strings that need to be used for namespace packages, # the first one is ``pkgutil``, the second ``pkg_resources``. options = ('declare_namespace(__name__)', 'extend_path(__path__') if options[0] in content or options[1] in content: # It is a namespace, now try to find the rest of the # modules on sys_path or whatever the search_path is. paths = set() for s in search_path: other = os.path.join(s, self.name.string_name) if os.path.isdir(other): paths.add(other) if paths: return list(paths) # TODO I'm not sure if this is how nested namespace # packages work. The tests are not really good enough to # show that. # Default to this. return [self._get_init_directory()]
def __init__(self, source=None, line=None, column=None, path=None, encoding='utf-8', source_path=None): if source_path is not None: warnings.warn("Use path instead of source_path.", DeprecationWarning) path = source_path self._source_path = path self.path = None if path is None else os.path.abspath(path) if source is None: with open(path) as f: source = f.read() lines = source.splitlines() or [''] if source and source[-1] == '\n': lines.append('') line = max(len(lines), 1) if line is None else line if not (0 < line <= len(lines)): raise ValueError('`line` parameter is not in a valid range.') line_len = len(lines[line - 1]) column = line_len if column is None else column if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column cache.clear_caches() debug.reset_time() self.source = common.source_to_unicode(source, encoding) self._user_context = UserContext(self.source, self._pos) self._parser = UserContextParser(self.source, path, self._pos, self._user_context) self._evaluator = Evaluator() debug.speed('init')
def extract(script, new_name): """ The `args` / `kwargs` params are the same as in `api.Script`. :param operation: The refactoring operation to execute. :type operation: str :type source: str :return: list of changed lines/changed files """ new_lines = common.source_to_unicode(script.source).splitlines() old_lines = new_lines[:] user_stmt = script._parser.user_stmt() # TODO care for multiline extracts dct = {} if user_stmt: pos = script._pos line_index = pos[0] - 1 arr, index = helpers.array_for_pos(user_stmt, pos) if arr is not None: start_pos = arr[index].start_pos end_pos = arr[index].end_pos # take full line if the start line is different from end line e = end_pos[1] if end_pos[0] == start_pos[0] else None start_line = new_lines[start_pos[0] - 1] text = start_line[start_pos[1]:e] for l in range(start_pos[0], end_pos[0] - 1): text += '\n' + l if e is None: end_line = new_lines[end_pos[0] - 1] text += '\n' + end_line[:end_pos[1]] # remove code from new lines t = text.lstrip() del_start = start_pos[1] + len(text) - len(t) text = t.rstrip() del_end = len(t) - len(text) if e is None: new_lines[end_pos[0] - 1] = end_line[end_pos[1] - del_end:] e = len(start_line) else: e = e - del_end start_line = start_line[:del_start] + new_name + start_line[e:] new_lines[start_pos[0] - 1] = start_line new_lines[start_pos[0]:end_pos[0] - 1] = [] # add parentheses in multiline case open_brackets = ['(', '[', '{'] close_brackets = [')', ']', '}'] if '\n' in text and not (text[0] in open_brackets and text[-1] == close_brackets[open_brackets.index(text[0])]): text = '(%s)' % text # add new line before statement indent = user_stmt.start_pos[1] new = "%s%s = %s" % (' ' * indent, new_name, text) new_lines.insert(line_index, new) dct[script.path] = script.path, old_lines, new_lines return Refactoring(dct)
def test_source_to_unicode_unicode_text(self): source = ( b"# vim: fileencoding=utf-8\n" b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n" ) actual = common.source_to_unicode(source) expected = source.decode('utf-8') assert actual == expected
def check_fs(path): with open(path, 'rb') as f: source = source_to_unicode(f.read()) if name in source: module_name = os.path.basename(path)[:-3] # Remove `.py`. module = _load_module(evaluator, path, source) add_module(evaluator, module_name, module) return module
def precache_parser(*files): for f in files: f = os.path.abspath(f) with open(f) as f_: source = f_.read() source = common.source_to_unicode(source, 'utf-8') parser = FastParser(source, f) cache.save_parser(f, None, parser, pickling=False)
def load(buildout_script): try: with open(buildout_script, 'rb') as f: source = common.source_to_unicode(f.read()) except IOError: debug.dbg('Error trying to read buildout_script: %s', buildout_script) return p = Parser(evaluator.grammar, source, buildout_script) cache.save_parser(buildout_script, None, p) return p.module
def load(source): if path is not None and path.endswith('.py'): if source is None: with open(path) as f: source = f.read() else: return compiled.load_module(path, name) p = path or name p = fast.FastParser(common.source_to_unicode(source), p) cache.save_parser(path, name, p) return p.module
def parent(self): obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: class FakeParent(pr.Base): parent = None # To avoid having no parent for NamePart. path = None names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: module = __import__(module_name) fake_name = helpers.FakeName(names, FakeParent()) parser_path = fake_name.names raw_module = get_module(self._value) try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(source, path[:-1]).module if not parser_path: return mod found = self._evaluator.eval_call_path(iter(parser_path), mod, None) if found: return found[0] debug.warning('Interpreter lookup for Python code failed %s', mod) module = compiled.CompiledObject(raw_module) if raw_module == builtins: # The builtins module is special and always cached. module = compiled.builtin return compiled.create(self._evaluator, self._value, module, module)
def load(buildout_script): try: with open(buildout_script, "rb") as f: source = common.source_to_unicode(f.read()) except IOError: debug.dbg("Error trying to read buildout_script: %s", buildout_script) return p = ParserWithRecovery(evaluator.grammar, source, buildout_script) save_parser(buildout_script, p) return p.module
def load(buildout_script): try: with open(buildout_script, 'rb') as f: source = common.source_to_unicode(f.read()) except IOError: debug.dbg('Error trying to read buildout_script: %s', buildout_script) return p = Parser(evaluator.grammar, source, buildout_script) cache.save_parser(buildout_script, p) return p.module
def __init__(self, source=None, line=None, column=None, path=None, encoding='utf-8', source_path=None, source_encoding=None, sys_path=None): if source_path is not None: warnings.warn( "Deprecated since version 0.7. Use path instead of source_path.", DeprecationWarning, stacklevel=2) path = source_path if source_encoding is not None: warnings.warn( "Deprecated since version 0.8. Use encoding instead of source_encoding.", DeprecationWarning, stacklevel=2) encoding = source_encoding self._orig_path = path # An empty path (also empty string) should always result in no path. self.path = os.path.abspath(path) if path else None if source is None: # TODO add a better warning than the traceback! with open(path, 'rb') as f: source = f.read() self._source = common.source_to_unicode(source, encoding) self._code_lines = common.splitlines(self._source) line = max(len(self._code_lines), 1) if line is None else line if not (0 < line <= len(self._code_lines)): raise ValueError('`line` parameter is not in a valid range.') line_len = len(self._code_lines[line - 1]) column = line_len if column is None else column if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column self._path = path cache.clear_time_caches() debug.reset_time() self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2]) if sys_path is None: venv = os.getenv('VIRTUAL_ENV') if venv: sys_path = list(get_venv_path(venv)) self._evaluator = Evaluator(self._grammar, sys_path=sys_path) debug.speed('init')
def __init__(self, source=None, line=None, column=None, path=None, encoding='utf-8', source_path=None, source_encoding=None, sys_path=None): if source_path is not None: warnings.warn("Use path instead of source_path.", DeprecationWarning) path = source_path if source_encoding is not None: warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning) encoding = source_encoding self._orig_path = path # An empty path (also empty string) should always result in no path. self.path = os.path.abspath(path) if path else None if source is None: # TODO add a better warning than the traceback! try: with open(path) as f: source = f.read() except UnicodeDecodeError: with open(path, encoding=encoding) as f: source = f.read() self._source = common.source_to_unicode(source, encoding) self._code_lines = common.splitlines(self._source) line = max(len(self._code_lines), 1) if line is None else line if not (0 < line <= len(self._code_lines)): raise ValueError('`line` parameter is not in a valid range.') line_len = len(self._code_lines[line - 1]) column = line_len if column is None else column if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column self._path = path cache.clear_time_caches() debug.reset_time() self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2]) if sys_path is None: venv = os.getenv('VIRTUAL_ENV') if venv: sys_path = list(get_venv_path(venv)) self._evaluator = Evaluator(self._grammar, sys_path=sys_path) debug.speed('init')
def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith(".py") and not dotted_path in settings.auto_import_modules: if source is None: with open(path, "rb") as f: source = f.read() else: return compiled.load_module(path) p = path p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) cache.save_parser(path, p) return p.module
def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith('.py') \ and not dotted_path in settings.auto_import_modules: if source is None: with open(path, 'rb') as f: source = f.read() else: return compiled.load_module(path, name) p = path or name p = fast.FastParser(common.source_to_unicode(source), p) cache.save_parser(path, name, p) return p.module
def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith(('.py', '.zip', '.egg')) \ and dotted_path not in settings.auto_import_modules: if source is None: with open(path, 'rb') as f: source = f.read() else: return compiled.load_module(evaluator, path) p = path p = FastParser(evaluator.grammar, source_to_unicode(source), p) save_parser(path, p) return p.module
def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith('.py') \ and not dotted_path in settings.auto_import_modules: if source is None: with open(path, 'rb') as f: source = f.read() else: return compiled.load_module(path, name) p = path or name p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) cache.save_parser(path, name, p) return p.module
def __init__(self, source=None, line=None, column=None, path=None, encoding='utf-8', source_path=None, source_encoding=None, sys_path=None): if source_path is not None: warnings.warn("Use path instead of source_path.", DeprecationWarning) path = source_path if source_encoding is not None: warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning) encoding = source_encoding self._orig_path = path self.path = None if path is None else os.path.abspath(path) if source is None: with open(path) as f: source = f.read() self.source = common.source_to_unicode(source, encoding) lines = common.splitlines(self.source) line = max(len(lines), 1) if line is None else line if not (0 < line <= len(lines)): raise ValueError('`line` parameter is not in a valid range.') line_len = len(lines[line - 1]) column = line_len if column is None else column if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column cache.clear_time_caches() debug.reset_time() self._grammar = load_grammar('grammar%s.%s' % sys.version_info[:2]) self._user_context = UserContext(self.source, self._pos) self._parser = UserContextParser(self._grammar, self.source, path, self._pos, self._user_context, self._parsed_callback) if sys_path is None: venv = os.getenv('VIRTUAL_ENV') if venv: sys_path = list(get_venv_path(venv)) self._evaluator = Evaluator(self._grammar, sys_path=sys_path) debug.speed('init')
def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith(('.py', '.zip', '.egg')) \ and dotted_path not in settings.auto_import_modules: if source is None: with open(path, 'rb') as f: source = f.read() else: return compiled.load_module(evaluator, path) p = path p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) save_parser(path, p) from jedi.evaluate.representation import ModuleWrapper return ModuleWrapper(evaluator, p.module, parent_module)
def inline(script): """ :type script: api.Script """ new_lines = common.splitlines(common.source_to_unicode(script.source)) dct = {} definitions = script.goto_assignments() with common.ignored(AssertionError): assert len(definitions) == 1 stmt = definitions[0]._definition usages = script.usages() inlines = [ r for r in usages if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos ] inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column), reverse=True) expression_list = stmt.expression_list() # don't allow multiline refactorings for now. assert stmt.start_pos[0] == stmt.end_pos[0] index = stmt.start_pos[0] - 1 line = new_lines[index] replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1] replace_str = replace_str.strip() # tuples need parentheses if expression_list and isinstance(expression_list[0], pr.Array): arr = expression_list[0] if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1: replace_str = '(%s)' % replace_str # if it's the only assignment, remove the statement if len(stmt.get_defined_names()) == 1: line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:] dct = _rename(inlines, replace_str) # remove the empty line new_lines = dct[script.path][2] if line.strip(): new_lines[index] = line else: new_lines.pop(index) return Refactoring(dct)
def parent(self): parser_path = [] obj = self._value if inspect.ismodule(obj): module = obj else: try: o = obj.__objclass__ parser_path.append( pr.NamePart(obj.__name__, None, (None, None))) obj = o except AttributeError: pass try: module_name = obj.__module__ parser_path.insert( 0, pr.NamePart(obj.__name__, None, (None, None))) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: module = __import__(module_name) raw_module = get_module(self._value) try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(source, path[:-1]).module if not parser_path: return mod found = self._evaluator.eval_call_path(iter(parser_path), mod, None) if found: return found[0] debug.warning('Interpreter lookup for Python code failed %s', mod) module = compiled.CompiledObject(raw_module) return compiled.create(self._value, module, module)
def defined_names(source, path=None, encoding='utf-8'): """ Get all definitions in `source` sorted by its position. This functions can be used for listing functions, classes and data defined in a file. This can be useful if you want to list them in "sidebar". Each element in the returned list also has `defined_names` method which can be used to get sub-definitions (e.g., methods in class). :rtype: list of classes.Definition """ parser = Parser( common.source_to_unicode(source, encoding), module_path=path, ) return classes.defined_names(Evaluator(), parser.module)
def __init__(self, source=None, line=None, column=None, path=None, encoding='utf-8', source_path=None, source_encoding=None): if source_path is not None: warnings.warn("Use path instead of source_path.", DeprecationWarning) path = source_path if source_encoding is not None: warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning) encoding = source_encoding self._orig_path = path self.path = None if path is None else os.path.abspath(path) if source is None: with open(path) as f: source = f.read() lines = source.splitlines() or [''] if source and source[-1] == '\n': lines.append('') line = max(len(lines), 1) if line is None else line if not (0 < line <= len(lines)): raise ValueError('`line` parameter is not in a valid range.') line_len = len(lines[line - 1]) column = line_len if column is None else column if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column cache.clear_caches() debug.reset_time() self.source = common.source_to_unicode(source, encoding) self._user_context = UserContext(self.source, self._pos) self._parser = UserContextParser(self.source, path, self._pos, self._user_context) self._evaluator = Evaluator() debug.speed('init')
def parent(self): parser_path = [] obj = self._value if inspect.ismodule(obj): module = obj else: try: o = obj.__objclass__ parser_path.append(pr.NamePart(obj.__name__, None, (None, None))) obj = o except AttributeError: pass try: module_name = obj.__module__ parser_path.insert(0, pr.NamePart(obj.__name__, None, (None, None))) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: module = __import__(module_name) raw_module = get_module(self._value) try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(source, path[:-1]).module if not parser_path: return mod found = self._evaluator.eval_call_path(iter(parser_path), mod, None) if found: return found[0] debug.warning('Interpreter lookup for Python code failed %s', mod) module = compiled.CompiledObject(raw_module) return compiled.create(self._value, module, module)
def inline(script): """ :type script: api.Script """ new_lines = common.splitlines(common.source_to_unicode(script.source)) dct = {} definitions = script.goto_assignments() with common.ignored(AssertionError): assert len(definitions) == 1 stmt = definitions[0]._definition usages = script.usages() inlines = [r for r in usages if not stmt.start_pos <= (r.line, r.column) <= stmt.end_pos] inlines = sorted(inlines, key=lambda x: (x.module_path, x.line, x.column), reverse=True) expression_list = stmt.expression_list() # don't allow multiline refactorings for now. assert stmt.start_pos[0] == stmt.end_pos[0] index = stmt.start_pos[0] - 1 line = new_lines[index] replace_str = line[expression_list[0].start_pos[1]:stmt.end_pos[1] + 1] replace_str = replace_str.strip() # tuples need parentheses if expression_list and isinstance(expression_list[0], pr.Array): arr = expression_list[0] if replace_str[0] not in ['(', '[', '{'] and len(arr) > 1: replace_str = '(%s)' % replace_str # if it's the only assignment, remove the statement if len(stmt.get_defined_names()) == 1: line = line[:stmt.start_pos[1]] + line[stmt.end_pos[1]:] dct = _rename(inlines, replace_str) # remove the empty line new_lines = dct[script.path][2] if line.strip(): new_lines[index] = line else: new_lines.pop(index) return Refactoring(dct)
def return_value(search_path): init_path = self.py__file__() if os.path.basename(init_path) == '__init__.py': with open(init_path, 'rb') as f: content = common.source_to_unicode(f.read()) # these are strings that need to be used for namespace packages, # the first one is ``pkgutil``, the second ``pkg_resources``. options = ('declare_namespace(__name__)', 'extend_path(__path__') if options[0] in content or options[1] in content: # It is a namespace, now try to find the rest of the # modules on sys_path or whatever the search_path is. paths = set() for s in search_path: other = os.path.join(s, unicode(self.name)) if os.path.isdir(other): paths.add(other) return list(paths) # Default to this. return [path]
def __init__(self, source=None, line=None, column=None, path=None, encoding='utf-8', source_path=None, source_encoding=None, resolve_variables_to_types=True): if source_path is not None: warnings.warn("Use path instead of source_path.", DeprecationWarning) path = source_path if source_encoding is not None: warnings.warn("Use encoding instead of source_encoding.", DeprecationWarning) encoding = source_encoding self._orig_path = path self.path = None if path is None else os.path.abspath(path) if source is None: with open(path) as f: source = f.read() self.source = common.source_to_unicode(source, encoding) lines = common.splitlines(self.source) line = max(len(lines), 1) if line is None else line if not (0 < line <= len(lines)): raise ValueError('`line` parameter is not in a valid range.') line_len = len(lines[line - 1]) column = line_len if column is None else column if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column if not cache.never_clear_cache: cache.clear_caches() debug.reset_time() self._user_context = UserContext(self.source, self._pos) self._parser = UserContextParser(self.source, path, self._pos, self._user_context) self._evaluator = Evaluator(resolve_variables_to_types=resolve_variables_to_types) debug.speed('init')
def parse(grammar, path): with open(path) as f: source = f.read() source = common.source_to_unicode(source) return FastParser(grammar, source, path)
def check_fs(path): with open(path) as f: source = source_to_unicode(f.read()) if name in source: return load_module(path, source)
def parent(self): """ Creating fake statements for the interpreter. Here we are trying to link back to Python code, if possible. This means we try to find the python module for a name (not the builtin). """ return mixed.create(self._evaluator, self._value) obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: # If we put anything into fromlist, it will just return the # latest name. module = __import__(module_name, fromlist=['']) parser_path = names found = [] try: path = module.__file__ except AttributeError: pass else: # Find the corresponding Python file for an interpreted one. path = re.sub(r'\.pyc$', '.py', path) if path.endswith('.py'): with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(load_grammar(), source, path).module mod = self._evaluator.wrap(mod) # We have to make sure that the modules that we import are also # part of evaluator.modules. for module_name, module in sys.modules.items(): try: iterated_path = module.__file__ except AttributeError: pass else: if iterated_path == path: self._evaluator.modules[module_name] = mod break else: raise NotImplementedError('This should not happen, a module ' 'should be part of sys.modules.') if parser_path: assert len(parser_path) == 1 found = list(self._evaluator.find_types(mod, parser_path[0], search_global=True)) else: found = [mod] if not found: debug.warning('Interpreter lookup failed in global scope for %s', parser_path) if not found: evaluated = compiled.create(self._evaluator, obj) found = [evaluated] if len(found) > 1: content = iterable.AlreadyEvaluated(found) stmt = pt.ExprStmt([self, pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content]) stmt.parent = self._module return stmt else: return found[0]
def parse(code=None, path=None, grammar=None, error_recovery=True, start_symbol='file_input', cache=False, diff_cache=False): """ If you want to parse a Python file you want to start here, most likely. If you need finer grained control over the parsed instance, there will be other ways to access it. :param code: A unicode string that contains Python code. :param path: The path to the file you want to open. Only needed for caching. :param grammar: A Python grammar file, created with load_grammar. You may not specify it. In that case it's the current Python version. :param error_recovery: If enabled, any code will be returned. If it is invalid, it will be returned as an error node. If disabled, you will get a ParseError when encountering syntax errors in your code. :param start_symbol: The grammar symbol that you want to parse. Only allowed to be used when error_recovery is disabled. :return: A syntax tree node. Typically the module. """ if code is None and path is None: raise TypeError("Please provide either code or a path.") if grammar is None: grammar = load_grammar() if cache and not code and path is not None: # In this case we do actual caching. We just try to load it. module_node = load_module(grammar, path) if module_node is not None: return module_node if code is None: with open(path, 'rb') as f: code = source_to_unicode(f.read()) if diff_cache and settings.fast_parser: try: module_cache_item = parser_cache[path] except KeyError: pass else: lines = splitlines(code, keepends=True) module_node = module_cache_item.node old_lines = module_cache_item.lines if old_lines == lines: save_module(grammar, path, module_node, lines, pickling=False) return module_node new_node = DiffParser(grammar, module_node).update( old_lines=old_lines, new_lines=lines ) save_module(grammar, path, new_node, lines, pickling=cache) return new_node added_newline = not code.endswith('\n') lines = tokenize_lines = splitlines(code, keepends=True) if added_newline: code += '\n' tokenize_lines = list(tokenize_lines) tokenize_lines[-1] += '\n' tokenize_lines.append('') tokens = generate_tokens(tokenize_lines, use_exact_op_types=True) p = Parser(grammar, error_recovery=error_recovery, start_symbol=start_symbol) root_node = p.parse(tokens=tokens) if added_newline: _remove_last_newline(root_node) if cache or diff_cache: save_module(grammar, path, root_node, lines, pickling=cache) return root_node
def parent(self): """ Creating fake statements for the interpreter. """ obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: # TODO this import is wrong. Yields x for x.y.z instead of z module = __import__(module_name) parser_path = names raw_module = get_module(self._value) found = [] try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(load_grammar(), source, path[:-1]).module if parser_path: assert len(parser_path) == 1 found = self._evaluator.find_types(mod, parser_path[0], search_global=True) else: found = [er.wrap(self._evaluator, mod)] if not found: debug.warning( 'Possibly an interpreter lookup for Python code failed %s', parser_path) if not found: evaluated = compiled.CompiledObject(obj) if evaluated == builtins: # The builtins module is special and always cached. evaluated = compiled.builtin found = [evaluated] content = iterable.AlreadyEvaluated(found) stmt = pt.ExprStmt([ self, pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content ]) stmt.parent = self._module return stmt
def parent(self): """ Creating fake statements for the interpreter. Here we are trying to link back to Python code, if possible. This means we try to find the python module for a name (not the builtin). """ return mixed.create(self._evaluator, self._value) obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: # If we put anything into fromlist, it will just return the # latest name. module = __import__(module_name, fromlist=['']) parser_path = names found = [] try: path = module.__file__ except AttributeError: pass else: # Find the corresponding Python file for an interpreted one. path = re.sub(r'\.pyc$', '.py', path) if path.endswith('.py'): with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(load_grammar(), source, path).module mod = self._evaluator.wrap(mod) # We have to make sure that the modules that we import are also # part of evaluator.modules. for module_name, module in sys.modules.items(): try: iterated_path = module.__file__ except AttributeError: pass else: if iterated_path == path: self._evaluator.modules[module_name] = mod break else: raise NotImplementedError( 'This should not happen, a module ' 'should be part of sys.modules.') if parser_path: assert len(parser_path) == 1 found = list( self._evaluator.find_types(mod, parser_path[0], search_global=True)) else: found = [mod] if not found: debug.warning( 'Interpreter lookup failed in global scope for %s', parser_path) if not found: evaluated = compiled.create(self._evaluator, obj) found = [evaluated] if len(found) > 1: content = iterable.AlreadyEvaluated(found) stmt = pt.ExprStmt([ self, pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content ]) stmt.parent = self._module return stmt else: return found[0]
def check_fs(path): with open(path, 'rb') as f: source = source_to_unicode(f.read()) if name in source: return _load_module(evaluator, path, source)
def parent(self): """ Creating fake statements for the interpreter. """ obj = self._value parser_path = [] if inspect.ismodule(obj): module = obj else: names = [] try: o = obj.__objclass__ names.append(obj.__name__) obj = o except AttributeError: pass try: module_name = obj.__module__ names.insert(0, obj.__name__) except AttributeError: # Unfortunately in some cases like `int` there's no __module__ module = builtins else: # TODO this import is wrong. Yields x for x.y.z instead of z module = __import__(module_name) parser_path = names raw_module = get_module(self._value) found = [] try: path = module.__file__ except AttributeError: pass else: path = re.sub('c$', '', path) if path.endswith('.py'): # cut the `c` from `.pyc` with open(path) as f: source = source_to_unicode(f.read()) mod = FastParser(load_grammar(), source, path[:-1]).module if parser_path: assert len(parser_path) == 1 found = self._evaluator.find_types(mod, parser_path[0], search_global=True) else: found = [er.wrap(self._evaluator, mod)] if not found: debug.warning('Possibly an interpreter lookup for Python code failed %s', parser_path) if not found: evaluated = compiled.CompiledObject(obj) if evaluated == builtins: # The builtins module is special and always cached. evaluated = compiled.builtin found = [evaluated] content = iterable.AlreadyEvaluated(found) stmt = pt.ExprStmt([self, pt.Operator(pt.zero_position_modifier, '=', (0, 0), ''), content]) stmt.parent = self._module return stmt
def parse(code=None, path=None, grammar=None, error_recovery=True, start_symbol='file_input', cache=False, diff_cache=False): """ If you want to parse a Python file you want to start here, most likely. If you need finer grained control over the parsed instance, there will be other ways to access it. :param code: A unicode string that contains Python code. :param path: The path to the file you want to open. Only needed for caching. :param grammar: A Python grammar file, created with load_grammar. :param error_recovery: If enabled, any code will be returned. If it is invalid, it will be returned as an error node. If disabled, you will get a ParseError when encountering syntax errors in your code. :param start_symbol: The grammar symbol that you want to parse. Only allowed to be used when error_recovery is disabled. :return: A syntax tree node. Typically the module. """ if code is None and path is None: raise TypeError("Please provide either code or a path.") if grammar is None: grammar = load_grammar() if path is not None: path = os.path.expanduser(path) if cache and not code and path is not None: # In this case we do actual caching. We just try to load it. module_node = load_module(grammar, path) if module_node is not None: return module_node if code is None: with open(path, 'rb') as f: code = source_to_unicode(f.read()) if diff_cache and settings.fast_parser: try: module_cache_item = parser_cache[path] except KeyError: pass else: lines = splitlines(code, keepends=True) module_node = module_cache_item.node old_lines = module_cache_item.lines if old_lines == lines: save_module(grammar, path, module_node, lines, pickling=False) return module_node new_node = DiffParser(grammar, module_node).update( old_lines=old_lines, new_lines=lines ) save_module(grammar, path, new_node, lines, pickling=cache) return new_node added_newline = not code.endswith('\n') lines = tokenize_lines = splitlines(code, keepends=True) if added_newline: code += '\n' tokenize_lines = list(tokenize_lines) tokenize_lines[-1] += '\n' tokenize_lines.append('') tokens = generate_tokens(tokenize_lines, use_exact_op_types=True) p = Parser(grammar, error_recovery=error_recovery, start_symbol=start_symbol) root_node = p.parse(tokens=tokens) if added_newline: _remove_last_newline(root_node) if cache or diff_cache: save_module(grammar, path, root_node, lines, pickling=cache) return root_node