def test_modulepickling_simulate_deleted_cache(tmpdir): """ Tests loading from a cache file after it is deleted. According to macOS `dev docs`__, Note that the system may delete the Caches/ directory to free up disk space, so your app must be able to re-create or download these files as needed. It is possible that other supported platforms treat cache files the same way. __ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html """ grammar = load_grammar() module = 'fake parser' # Create the file path = tmpdir.dirname + '/some_path' with open(path, 'w'): pass io = file_io.FileIO(path) save_module(grammar._hashed, io, module, lines=[]) assert load_module(grammar._hashed, io) == module unlink(_get_hashed_path(grammar._hashed, path)) parser_cache.clear() cached2 = load_module(grammar._hashed, io) assert cached2 is None
def test_modulepickling_simulate_deleted_cache(tmpdir): """ Tests loading from a cache file after it is deleted. According to macOS `dev docs`__, Note that the system may delete the Caches/ directory to free up disk space, so your app must be able to re-create or download these files as needed. It is possible that other supported platforms treat cache files the same way. __ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html """ grammar = load_grammar() module = 'fake parser' # Create the file path = tmpdir.dirname + '/some_path' with open(path, 'w'): pass save_module(grammar._hashed, path, module, []) assert load_module(grammar._hashed, path) == module unlink(_get_hashed_path(grammar._hashed, path)) parser_cache.clear() cached2 = load_module(grammar._hashed, path) assert cached2 is None
def _parse(self, code=None, error_recovery=True, path=None, start_symbol=None, cache=False, diff_cache=False, cache_path=None, start_pos=(1, 0)): """ Wanted python3.5 * operator and keyword only arguments. Therefore just wrap it all. start_pos here is just a parameter internally used. Might be public sometime in the future. """ if code is None and path is None: raise TypeError("Please provide either code or a path.") if start_symbol is None: start_symbol = self._start_symbol if error_recovery and start_symbol != 'file_input': raise NotImplementedError("This is currently not implemented.") if cache and code is None and path is not None: # With the current architecture we cannot load from cache if the # code is given, because we just load from cache if it's not older than # the latest change (file last modified). module_node = load_module(self._hashed, path, cache_path=cache_path) if module_node is not None: return module_node if code is None: with open(path, 'rb') as f: code = f.read() code = python_bytes_to_unicode(code) lines = split_lines(code, keepends=True) if diff_cache: if self._diff_parser is None: raise TypeError("You have to define a diff parser to be able " "to use this option.") try: module_cache_item = parser_cache[self._hashed][path] except KeyError: pass else: module_node = module_cache_item.node old_lines = module_cache_item.lines if old_lines == lines: return module_node new_node = self._diff_parser( self._pgen_grammar, self._tokenizer, module_node).update(old_lines=old_lines, new_lines=lines) save_module( self._hashed, path, new_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return new_node tokens = self._tokenizer(lines, start_pos) p = self._parser(self._pgen_grammar, error_recovery=error_recovery, start_symbol=start_symbol) root_node = p.parse(tokens=tokens) if cache or diff_cache: save_module( self._hashed, path, root_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return root_node
def _parse(self, code=None, error_recovery=True, path=None, start_symbol=None, cache=False, diff_cache=False, cache_path=None, file_io=None, start_pos=(1, 0)): """ Wanted python3.5 * operator and keyword only arguments. Therefore just wrap it all. start_pos here is just a parameter internally used. Might be public sometime in the future. """ if code is None and path is None and file_io is None: raise TypeError("Please provide either code or a path.") if start_symbol is None: start_symbol = self._start_nonterminal if error_recovery and start_symbol != 'file_input': raise NotImplementedError("This is currently not implemented.") if file_io is None: if code is None: file_io = FileIO(path) else: file_io = KnownContentFileIO(path, code) if cache and file_io.path is not None: module_node = load_module(self._hashed, file_io, cache_path=cache_path) if module_node is not None: return module_node if code is None: code = file_io.read() code = python_bytes_to_unicode(code) lines = split_lines(code, keepends=True) if diff_cache: if self._diff_parser is None: raise TypeError("You have to define a diff parser to be able " "to use this option.") try: module_cache_item = parser_cache[self._hashed][file_io.path] except KeyError: pass else: module_node = module_cache_item.node old_lines = module_cache_item.lines if old_lines == lines: return module_node new_node = self._diff_parser( self._pgen_grammar, self._tokenizer, module_node ).update( old_lines=old_lines, new_lines=lines ) save_module(self._hashed, file_io, new_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return new_node tokens = self._tokenizer(lines, start_pos) p = self._parser( self._pgen_grammar, error_recovery=error_recovery, start_nonterminal=start_symbol ) root_node = p.parse(tokens=tokens) if cache or diff_cache: save_module(self._hashed, file_io, root_node, lines, # Never pickle in pypy, it's slow as hell. pickling=cache and not is_pypy, cache_path=cache_path) return root_node