def _from_loader(loader, string): is_package = loader.is_package(string) try: get_filename = loader.get_filename except AttributeError: return None, is_package else: module_path = cast_path(get_filename(string)) # To avoid unicode and read bytes, "overwrite" loader.get_source if # possible. f = type(loader).get_source if is_py3 and f is not importlib.machinery.SourceFileLoader.get_source: # Unfortunately we are reading unicode here, not bytes. # It seems hard to get bytes, because the zip importer # logic just unpacks the zip file and returns a file descriptor # that we cannot as easily access. Therefore we just read it as # a string in the cases where get_source was overwritten. code = loader.get_source(string) else: code = _get_source(loader, string) if code is None: return None, is_package if isinstance(loader, zipimporter): return ZipFileIO(module_path, code, cast_path(loader.archive)), is_package return KnownContentFileIO(module_path, code), is_package
def _load_module_from_path(evaluator, path, base_names, code): """ This should pretty much only be used for get_modules_containing_name. It's here to ensure that a random path is still properly loaded into the Jedi module structure. """ e_sys_path = evaluator.get_sys_path() if base_names: module_name = os.path.basename(path) module_name = sys_path.remove_python_path_suffix(module_name) is_package = module_name == '__init__' if is_package: import_names = base_names else: import_names = base_names + (module_name, ) else: import_names, is_package = sys_path.transform_path_to_dotted( e_sys_path, path) module = _load_python_module( evaluator, KnownContentFileIO(path, code), sys_path=e_sys_path, import_names=import_names, is_package=is_package, ) evaluator.module_cache.add(import_names, ContextSet([module])) return module
def find_module_pre_py3(string, path=None, full_name=None, is_global_search=True): # This import is here, because in other places it will raise a # DeprecationWarning. import imp try: module_file, module_path, description = imp.find_module(string, path) module_type = description[2] is_package = module_type is imp.PKG_DIRECTORY if is_package: # In Python 2 directory package imports are returned as folder # paths, not __init__.py paths. p = os.path.join(module_path, '__init__.py') try: module_file = open(p) module_path = p except FileNotFoundError: pass elif module_type != imp.PY_SOURCE: if module_file is not None: module_file.close() module_file = None if module_file is None: code = None return None, is_package with module_file: code = module_file.read() return KnownContentFileIO(cast_path(module_path), code), is_package except ImportError: pass if path is None: path = sys.path for item in path: loader = pkgutil.get_importer(item) if loader: loader = loader.find_module(string) if loader is not None: return _from_loader(loader, string) raise ImportError("No module named {}".format(string))
def _parse(self, code=None, error_recovery=True, path=None, start_symbol=None, cache=False, diff_cache=False, cache_path=None, file_io=None, start_pos=(1, 0)): """ Wanted python3.5 * operator and keyword only arguments. Therefore just wrap it all. start_pos here is just a parameter internally used. Might be static sometime in the future. """ if code is None and path is None and file_io is None: raise TypeError("Please provide either code or a path.") if start_symbol is None: start_symbol = self._start_nonterminal if error_recovery and start_symbol != 'file_input': raise NotImplementedError("This is currently not implemented.") if file_io is None: if code is None: file_io = FileIO(path) else: file_io = KnownContentFileIO(path, code) if cache and file_io.path is not None: module_node = load_module(self._hashed, file_io, cache_path=cache_path) if module_node is not None: return module_node if code is None: code = file_io.read() code = python_bytes_to_unicode(code) lines = split_lines(code, keepends=True) if diff_cache: if self._diff_parser is None: raise TypeError("You have to define a diff parser to be able " "to use this option.") try: module_cache_item = parser_cache[self._hashed][file_io.path] except KeyError: pass else: module_node = module_cache_item.node old_lines = module_cache_item.lines if old_lines == lines: return module_node new_node = self._diff_parser( self._pgen_grammar, self._tokenizer, module_node).update(old_lines=old_lines, new_lines=lines) try_to_save_module( self._hashed, file_io, new_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return new_node tokens = self._tokenizer(lines, start_pos=start_pos) p = self._parser(self._pgen_grammar, error_recovery=error_recovery, start_nonterminal=start_symbol) root_node = p.parse(tokens=tokens) if cache or diff_cache: try_to_save_module( self._hashed, file_io, root_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return root_node
def parse(self, code: Union[str, bytes] = None, *, error_recovery=True, path: Union[os.PathLike, str] = None, start_symbol: str = None, cache=False, diff_cache=False, cache_path: Union[os.PathLike, str] = None, file_io: FileIO = None) -> _NodeT: """ If you want to parse a Python file you want to start here, most likely. If you need finer grained control over the parsed instance, there will be other ways to access it. :param str code: A unicode or bytes string. When it's not possible to decode bytes to a string, returns a :py:class:`UnicodeDecodeError`. :param bool error_recovery: If enabled, any code will be returned. If it is invalid, it will be returned as an error node. If disabled, you will get a ParseError when encountering syntax errors in your code. :param str start_symbol: The grammar rule (nonterminal) that you want to parse. Only allowed to be used when error_recovery is False. :param str path: The path to the file you want to open. Only needed for caching. :param bool cache: Keeps a copy of the parser tree in RAM and on disk if a path is given. Returns the cached trees if the corresponding files on disk have not changed. Note that this stores pickle files on your file system (e.g. for Linux in ``~/.cache/parso/``). :param bool diff_cache: Diffs the cached python module against the new code and tries to parse only the parts that have changed. Returns the same (changed) module that is found in cache. Using this option requires you to not do anything anymore with the cached modules under that path, because the contents of it might change. This option is still somewhat experimental. If you want stability, please don't use it. :param bool cache_path: If given saves the parso cache in this directory. If not given, defaults to the default cache places on each platform. :return: A subclass of :py:class:`parso.tree.NodeOrLeaf`. Typically a :py:class:`parso.python.tree.Module`. """ if code is None and path is None and file_io is None: raise TypeError("Please provide either code or a path.") if isinstance(path, str): path = Path(path) if isinstance(cache_path, str): cache_path = Path(cache_path) if start_symbol is None: start_symbol = self._start_nonterminal if error_recovery and start_symbol != 'file_input': raise NotImplementedError("This is currently not implemented.") if file_io is None: if code is None: file_io = FileIO(path) # type: ignore else: file_io = KnownContentFileIO(path, code) if cache and file_io.path is not None: module_node = load_module(self._hashed, file_io, cache_path=cache_path) if module_node is not None: return module_node # type: ignore if code is None: code = file_io.read() code = python_bytes_to_unicode(code) lines = split_lines(code, keepends=True) if diff_cache: if self._diff_parser is None: raise TypeError("You have to define a diff parser to be able " "to use this option.") try: module_cache_item = parser_cache[self._hashed][file_io.path] except KeyError: pass else: module_node = module_cache_item.node old_lines = module_cache_item.lines if old_lines == lines: return module_node # type: ignore new_node = self._diff_parser( self._pgen_grammar, self._tokenizer, module_node).update(old_lines=old_lines, new_lines=lines) try_to_save_module( self._hashed, file_io, new_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return new_node # type: ignore tokens = self._tokenizer(lines) p = self._parser(self._pgen_grammar, error_recovery=error_recovery, start_nonterminal=start_symbol) root_node = p.parse(tokens=tokens) if cache or diff_cache: try_to_save_module( self._hashed, file_io, root_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return root_node # type: ignore