def test_errors_added_during_AST_transformation(self): parser_fac = grammar_provider(ARITHMETIC_EBNF) trans_fac = lambda : partial(traverse, processing_table=self.trans_table) errata = grammar_unit(self.cases1, parser_fac, trans_fac, 'REPORT_ASTFailureTest') assert "marked with error" in str(errata) errata = grammar_unit(self.cases2, parser_fac, trans_fac, 'REPORT_ASTFailureTest') assert not errata
def test_fail_failtest(self): """Failure test should not pass if it failed because the parser is unknown.""" fcases = {} fcases['berm'] = {} fcases['berm']['fail'] = self.failure_cases['term']['fail'] errata = grammar_unit(fcases, grammar_provider(ARITHMETIC_EBNF), lambda : ARITHMETIC_EBNFTransform, 'REPORT_TestGrammarTest') assert errata
def test_testing_grammar(self): parser_fac = grammar_provider(ARITHMETIC_EBNF) trans_fac = lambda : ARITHMETIC_EBNFTransform # reset_unit(self.cases) errata = grammar_unit(self.cases, parser_fac, trans_fac, 'REPORT_TestGrammarTest') assert errata, "Unknown parser, but no error message!?" report = get_report(self.cases) assert report.find('### CST') >= 0 errata = grammar_unit(self.failure_cases, parser_fac, trans_fac, 'REPORT_TestGrammarTest') assert len(errata) == 3, str(errata)
def test_compileEBNF(self): parser_src = compileEBNF(ARITHMETIC_EBNF) assert isinstance(parser_src, str), str(type(parser_src)) assert parser_src.find('DSLGrammar') >= 0 parser_src = compileEBNF(ARITHMETIC_EBNF, branding="CustomDSL") assert isinstance(parser_src, str), str(type(parser_src)) assert parser_src.find('CustomDSLGrammar') >= 0 factory = grammar_provider(ARITHMETIC_EBNF, branding="TestDSL") assert callable(factory) parser = factory() result = parser("5 + 3 * 4") assert not is_error(result.error_flag), str(result.errors_sorted) result = parser("5A + 4B ** 4C") assert is_error(result.error_flag)
def test_equality2(self): ebnf = '@literalws = right\nterm = term ("*"|"/") factor | factor\nfactor = /[0-9]+/~' att = { "term": [ remove_empty, remove_whitespace, replace_by_single_child, flatten ], "factor": [remove_empty, remove_whitespace, reduce_single_child], "*": [remove_empty, remove_whitespace, replace_by_single_child] } parser = grammar_provider(ebnf)() tree = parser("20 / 4 * 3") traverse(tree, att) compare_tree = parse_sxpr( "(term (term (factor 20) (:Text /) (factor 4)) (:Text *) (factor 3))" ) assert tree.equals(compare_tree), tree.as_sxpr()
class TestLookahead: """ Testing of Expressions with trailing Lookahead-Parser. """ EBNF = r""" document = { category | entry } { LF } category = { LF } sequence_of_letters { /:/ sequence_of_letters } /:/ §&(LF sequence_of_letters) entry = { LF } sequence_of_letters !/:/ sequence_of_letters = /[A-Za-z0-9 ]+/ LF = / *\n/ """ cases = { "category": { "match": { 1: """Mountains: big: K2""", # case 1: matches only with lookahead (but should not fail in a test) 2: """Rivers:""", # case 2: lookahaead failure occurs at end of file and is mandatory. (should not fail as a test) 3: """Mountains: big:""" # same here }, "fail": { 6: """Mountains: big: """ } } } fail_cases = { "category": { "match": { 1: """Mountains: b""", # stop sign ":" is missing 2: """Rivers: # not allowed""", 2: """Mountains: K2 Rivers:""" # lookahead only covers K2 }, "fail": { 1: """Mountains: big: K2""" } } } grammar_fac = grammar_provider(EBNF) trans_fac = lambda: partial(traverse, processing_table={"*": [flatten, remove_empty]}) def setup(self): self.save_dir = os.getcwd() os.chdir(scriptpath) def teardown(self): clean_report('REPORT_TestLookahead') os.chdir(self.save_dir) def test_selftest(self): doc = """ Mountains: big: Mount Everest K2 Mountains: medium: Denali Alpomayo Rivers: Nile """ grammar = TestLookahead.grammar_fac() cst = grammar(doc) assert not cst.error_flag def test_unit_lookahead(self): gr = TestLookahead.grammar_fac() set_tracer(gr, trace_history) # Case 1: Lookahead string is part of the test case; parser fails but for the lookahead result = gr(self.cases['category']['match'][1], 'category') assert any(e.code in (PARSER_LOOKAHEAD_FAILURE_ONLY, PARSER_LOOKAHEAD_MATCH_ONLY) for e in result.errors), str(result.errors) # Case 2: Lookahead string is not part of the test case; parser matches but for the mandatory continuation result = gr(self.cases['category']['match'][2], 'category') assert any(e.code == MANDATORY_CONTINUATION_AT_EOF for e in result.errors) errata = grammar_unit(self.cases, TestLookahead.grammar_fac, TestLookahead.trans_fac, 'REPORT_TestLookahead') assert not errata, str(errata) errata = grammar_unit(self.fail_cases, TestLookahead.grammar_fac, TestLookahead.trans_fac, 'REPORT_TestLookahead') assert errata
class TestTokenParsing: ebnf = r""" @ tokens = BEGIN_INDENT, END_INDENT @ whitespace = /[ \t]*/ block = { line | indentBlock }+ line = ~/[^\x1b\x1c\x1d\n]*\n/ indentBlock = BEGIN_INDENT block END_INDENT """ set_config_value('max_parser_dropouts', 3) grammar = grammar_provider(ebnf)() code = lstrip_docstring(""" def func(x, y): if x > 0: # a comment if y > 0: print(x) # another comment print(y) """) tokenized, _ = tokenize_indentation(code) srcmap = tokenized_to_original_mapping(tokenized, code) def verify_mapping(self, teststr, orig_text, preprocessed_text, mapping): mapped_pos = preprocessed_text.find(teststr) assert mapped_pos >= 0 file_name, file_content, original_pos = mapping(mapped_pos) # original_pos = source_map(mapped_pos, self.srcmap) assert orig_text[original_pos:original_pos + len(teststr)] == teststr, \ '"%s" (%i) wrongly mapped onto "%s" (%i)' % \ (teststr, mapped_pos, orig_text[original_pos:original_pos + len(teststr)], original_pos) def test_strip_tokens(self): assert self.code == strip_tokens(self.tokenized) def test_parse_tokenized(self): cst = self.grammar(self.tokenized) assert not cst.error_flag def test_source_mapping_1(self): mapping = partial(source_map, srcmap=self.srcmap) self.verify_mapping("def func", self.code, self.tokenized, mapping) self.verify_mapping("x > 0:", self.code, self.tokenized, mapping) self.verify_mapping("if y > 0:", self.code, self.tokenized, mapping) self.verify_mapping("print(x)", self.code, self.tokenized, mapping) self.verify_mapping("print(y)", self.code, self.tokenized, mapping) def test_source_mapping_2(self): previous_index = 0 L = len(self.code) for mapped_index in range(len(self.tokenized)): _, _, index = source_map(mapped_index, self.srcmap) assert previous_index <= index <= L, \ "%i <= %i <= %i violated" % (previous_index, index, L) previous_index = index def test_non_token_preprocessor(self): _, tokenized, mapping, _ = preprocess_comments(self.code, 'no_uri') self.verify_mapping("def func", self.code, tokenized, mapping) self.verify_mapping("x > 0:", self.code, tokenized, mapping) self.verify_mapping("if y > 0:", self.code, tokenized, mapping) self.verify_mapping("print(x)", self.code, tokenized, mapping) self.verify_mapping("print(y)", self.code, tokenized, mapping) def test_chained_preprocessors(self): pchain = chain_preprocessors(preprocess_comments, preprocess_indentation) _, tokenized, mapping, _ = pchain(self.code, 'no_uri') self.verify_mapping("def func", self.code, tokenized, mapping) self.verify_mapping("x > 0:", self.code, tokenized, mapping) self.verify_mapping("if y > 0:", self.code, tokenized, mapping) self.verify_mapping("print(x)", self.code, tokenized, mapping) self.verify_mapping("print(y)", self.code, tokenized, mapping) def test_error_position(self): orig_src = self.code.replace('#', '\x1b') prepr = chain_preprocessors(preprocess_comments, preprocess_indentation) self.grammar.max_parser_dropouts__ = 3 result, messages, syntaxtree = compile_source(orig_src, prepr, self.grammar, lambda i: i, lambda i: i) for err in messages: if self.code[err.orig_pos] == "#": break else: assert False, "wrong error positions"