def test_simple(): """ The diff parser reuses modules. So check for that. """ grammar = load_grammar() module_a = grammar.parse('a', diff_cache=True) assert grammar.parse('b', diff_cache=True) == module_a
def test_modulepickling_simulate_deleted_cache(tmpdir): """ Tests loading from a cache file after it is deleted. According to macOS `dev docs`__, Note that the system may delete the Caches/ directory to free up disk space, so your app must be able to re-create or download these files as needed. It is possible that other supported platforms treat cache files the same way. __ https://developer.apple.com/library/content/documentation/FileManagement/Conceptual/FileSystemProgrammingGuide/FileSystemOverview/FileSystemOverview.html """ grammar = load_grammar() module = 'fake parser' # Create the file path = tmpdir.dirname + '/some_path' with open(path, 'w'): pass io = file_io.FileIO(path) save_module(grammar._hashed, io, module, lines=[]) assert load_module(grammar._hashed, io) == module unlink(_get_hashed_path(grammar._hashed, path)) parser_cache.clear() cached2 = load_module(grammar._hashed, io) assert cached2 is None
def test_load_grammar(): grammar = load_grammar(language="demo")._pgen_grammar assert grammar.start_nonterminal == 'file_input' nonterminals = {'file_input', 'rule', 'rhs', 'items', 'item', 'atom'} assert nonterminals == {x for x in grammar.nonterminal_to_dfas.keys()} reserved = {':', '|', '[', '*', '+', ']', '(', ')', 'ademo'} assert reserved == {x for x in grammar.reserved_syntax_strings.keys()}
def test_parse_demo_lang_error(): code = "abc : def + 'ss'\na" grammar = load_grammar(language="demo") root = grammar.parse(code) assert root.children[-2].get_code() == 'a' assert root.children[-1].type == 'endmarker' with pytest.raises(marso.parser.ParserSyntaxError): grammar.parse(code, error_recovery=False)
def test_on_itself(each_version): """ There are obviously no syntax erros in the Python code of marso. However marso should output the same for all versions. """ grammar = marso.load_grammar(version=each_version) path = os.path.dirname(os.path.dirname(__file__)) + '/marso' for file in get_python_files(path): tree = grammar.parse(path=file) errors = list(grammar.iter_errors(tree)) assert not errors
def test_py_LL1(): grammar = load_grammar(version="3.8") st1 = grammar.parse("print('OK') if (a := 1) > 0 else None", error_recovery=False) # https://www.python.org/dev/peps/pep-0617/#some-rules-are-not-actually-ll-1 # namedexpr_test: test [':=' test] # should be: # namedexpr_test: NAME [':=' test] # so as to reject following code: st2 = grammar.parse( "print('OK') if ([x for x in y] := [1,2,3]) > 0 else None", error_recovery=False)
def test_invalid_token_in_fstr(): module = load_grammar(version='3.6').parse('f"{a + ? + b}"') error_node, q, plus_b, error1, error2, endmarker = module.children assert error_node.get_code() == 'f"{a +' assert q.value == '?' assert q.type == 'error_leaf' assert plus_b.type == 'error_node' assert plus_b.get_code() == ' + b' assert error1.value == '}' assert error1.type == 'error_leaf' assert error2.value == '"' assert error2.type == 'error_leaf'
def test_dedent_issues1(): code = dedent('''\ class C: @property f g end ''') module = load_grammar(version='3.8').parse(code) klass, endmarker = module.children suite = klass.children[-1] assert suite.children[2].type == 'error_leaf' assert suite.children[3].get_code(include_prefix=False) == 'f\n' assert suite.children[5].get_code(include_prefix=False) == 'g\n' assert suite.type == 'suite'
def test_dedent_issues3(): code = dedent('''\ class C: f g ''') module = load_grammar(version='3.8').parse(code) klass, endmarker = module.children suite = klass.children[-1] assert len(suite.children) == 4 assert suite.children[1].get_code() == ' f\n' assert suite.children[1].type == 'simple_stmt' assert suite.children[2].get_code() == '' assert suite.children[2].type == 'error_leaf' assert suite.children[2].token_type == 'ERROR_DEDENT' assert suite.children[3].get_code() == ' g\n' assert suite.children[3].type == 'simple_stmt'
class Differ(object): grammar = load_grammar() def initialize(self, code): logging.debug('differ: initialize') try: del cache.parser_cache[self.grammar._hashed][None] except KeyError: pass self.lines = split_lines(code, keepends=True) self.module = parse(code, diff_cache=True, cache=True) assert code == self.module.get_code() _assert_valid_graph(self.module) return self.module def parse(self, code, copies=0, parsers=0, expect_error_leaves=False): logging.debug('differ: parse copies=%s parsers=%s', copies, parsers) lines = split_lines(code, keepends=True) diff_parser = DiffParser( self.grammar._pgen_grammar, self.grammar._tokenizer, self.module, ) new_module = diff_parser.update(self.lines, lines) self.lines = lines assert code == new_module.get_code() _assert_valid_graph(new_module) without_diff_parser_module = parse(code) _assert_nodes_are_equal(new_module, without_diff_parser_module) error_node = _check_error_leaves_nodes(new_module) assert expect_error_leaves == (error_node is not None), error_node if parsers is not ANY: assert diff_parser._parser_count == parsers if copies is not ANY: assert diff_parser._copy_count == copies return new_module
def test_parse_demo_language(): code = "abc : def + 'ss'\n" grammar = load_grammar(language="demo") root = grammar.parse(code) assert root.start_pos == (1, 0) assert root.end_pos == (2, 0) assert root.parent is None assert root.type == "file_input" assert root.children.__len__() == 2 assert root.children[0].type == "rule" assert root.children[1].type == "endmarker" rule = root.children[0] assert rule.parent == root assert rule.children.__len__() == 4 assert rule.children[0].type == "name" assert rule.children[0].value == "abc" assert rule.children[1].type == "operator" assert rule.children[1].value == ":" assert rule.children[1].start_pos == (1, 4) assert rule.children[1].prefix == " " assert rule.children[2].type == "items" assert rule.children[3].type == "newline" assert rule.children[3].value == "\n" items = rule.children[2] assert items.parent == rule assert items.children.__len__() == 2 assert items.children[0].type == "item" assert items.children[1].type == "string" assert items.children[1].value == "'ss'" item = items.children[0] assert item.children[0].type == "name" assert item.children[0].value == "def" assert item.children[1].type == "operator" assert item.children[1].value == "+"
def test_modulepickling_change_cache_dir(tmpdir): """ ParserPickling should not save old cache when cache_directory is changed. See: `#168 <https://github.com/davidhalter/jedi/pull/168>`_ """ dir_1 = str(tmpdir.mkdir('first')) dir_2 = str(tmpdir.mkdir('second')) item_1 = _NodeCacheItem('bla', []) item_2 = _NodeCacheItem('bla', []) path_1 = 'fake path 1' path_2 = 'fake path 2' hashed_grammar = load_grammar()._hashed _save_to_file_system(hashed_grammar, path_1, item_1, cache_path=dir_1) parser_cache.clear() cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_1) assert cached == item_1.node _save_to_file_system(hashed_grammar, path_2, item_2, cache_path=dir_2) cached = load_stored_item(hashed_grammar, path_1, item_1, cache_path=dir_2) assert cached is None
def test_dedent_issues2(): code = dedent('''\ class C: @property if 1: g else: h end ''') module = load_grammar(version='3.8').parse(code) klass, endmarker = module.children suite = klass.children[-1] assert suite.children[2].type == 'error_leaf' if_ = suite.children[3] assert if_.children[0] == 'if' assert if_.children[3].type == 'suite' assert if_.children[3].get_code() == '\n g\n' assert if_.children[4] == 'else' assert if_.children[6].type == 'suite' assert if_.children[6].get_code() == '\n h\n' assert suite.children[4].get_code(include_prefix=False) == 'end\n' assert suite.type == 'suite'
def __init__(self, project, environment=None, script_path=None): if environment is None: environment = project.get_environment() self.environment = environment self.script_path = script_path self.compiled_subprocess = environment.get_inference_state_subprocess(self) self.grammar = environment.get_grammar() # TTODO: this grammar is incorrect. self.latest_grammar = marso.load_grammar(version='3.7') self.memoize_cache = {} # for memoize decorators self.module_cache = imports.ModuleCache() # does the job of `sys.modules`. self.stub_module_cache = {} # Dict[Tuple[str, ...], Optional[ModuleValue]] self.compiled_cache = {} # see `inference.compiled.create()` self.inferred_element_counts = {} self.mixed_cache = {} # see `inference.compiled.mixed._create()` self.analysis = [] self.dynamic_params_depth = 0 self.is_analysis = False self.project = project self.access_cache = {} self.allow_descriptor_getattr = False self.flow_analysis_enabled = True self.reset_recursion_limitations()
def get_grammar(self): version_string = '%s.%s' % (self.version_info.major, self.version_info.minor) return marso.load_grammar(version=version_string)
def _get_error_list(code, version=None): grammar = marso.load_grammar(version=version) tree = grammar.parse(code) return list(grammar.iter_errors(tree))
def test_parse_py_error(): code = "abc : def + 'ss'\na" grammar = load_grammar(language="python") root = grammar.parse(code)
def __init__(self, code=None, line=None, column=None, path=None, encoding=None, sys_path=None, environment=None, project=None, source=None, language="python"): self._orig_path = path # An empty path (also empty string) should always result in no path. self.path = os.path.abspath(path) if path else None self.language = language self.grammar = marso.load_grammar(language=self.language) if encoding is None: encoding = 'utf-8' else: warnings.warn( "Deprecated since version 0.17.0. You should cast to valid " "unicode yourself, especially if you are not using utf-8.", DeprecationWarning, stacklevel=2 ) if line is not None: warnings.warn( "Providing the line is now done in the functions themselves " "like `Script(...).complete(line, column)`", DeprecationWarning, stacklevel=2 ) if column is not None: warnings.warn( "Providing the column is now done in the functions themselves " "like `Script(...).complete(line, column)`", DeprecationWarning, stacklevel=2 ) if source is not None: code = source warnings.warn( "Use the code keyword argument instead.", DeprecationWarning, stacklevel=2 ) if code is None: # TODO add a better warning than the traceback! with open(path, 'rb') as f: code = f.read() if sys_path is not None and not is_py3: sys_path = list(map(force_unicode, sys_path)) if project is None: # Load the Python grammar of the current interpreter. project = get_default_project( self.language, os.path.dirname(self.path) if path else None ) # TODO deprecate and remove sys_path from the Script API. if sys_path is not None: project._sys_path = sys_path warnings.warn( "Deprecated since version 0.17.0. Use the project API instead, " "which means Script(project=Project(dir, sys_path=sys_path)) instead.", DeprecationWarning, stacklevel=2 ) self._inference_state = InferenceState( project, environment=environment, script_path=self.path ) debug.speed('init') self._module_node, code = self._inference_state.parse_and_get_code( code=code, path=self.path, encoding=encoding, use_latest_grammar=path and path.endswith('.pyi'), cache=False, # No disk cache, because the current script often changes. diff_cache=settings.fast_parser, cache_path=settings.cache_directory, ) debug.speed('parsed') self._code_lines = marso.split_lines(code, keepends=True) self._code = code self._pos = line, column cache.clear_time_caches() debug.reset_time()
from medi._compatibility import unicode, is_py3 from medi.api.classes import Name from medi.api.completion import get_user_context from medi import parser_utils from medi.api.environment import get_default_environment, get_system_environment from medi.inference.gradual.conversion import convert_values from medi.inference.analysis import Warning TEST_COMPLETIONS = 0 TEST_INFERENCE = 1 TEST_GOTO = 2 TEST_REFERENCES = 3 grammar37 = marso.load_grammar(version='3.7') class BaseTestCase(object): def __init__(self, skip_version_info=None): self._skip_version_info = skip_version_info self._skip = None def set_skip(self, reason): self._skip = reason def get_skip_reason(self, environment): if self._skip is not None: return self._skip if self._skip_version_info is None:
def issues(code): grammar = marso.load_grammar() module = marso.parse(code) return grammar._get_normalizer_issues(module)
def __init__(self, version, is_passing): self.version = version self._is_passing = is_passing self.grammar = marso.load_grammar(version=self.version)
def _parse(code, version=None): code = dedent(code) + "\n\n" grammar = load_grammar(version=version) return grammar.parse(code, error_recovery=False)
def grammar(): return load_grammar(version='3.8')