def test_errors_added_during_AST_transformation(self):
parser_fac = grammar_provider(ARITHMETIC_EBNF)
trans_fac = lambda : partial(traverse, processing_table=self.trans_table)
errata = grammar_unit(self.cases1, parser_fac, trans_fac, 'REPORT_ASTFailureTest')
assert "marked with error" in str(errata)
errata = grammar_unit(self.cases2, parser_fac, trans_fac, 'REPORT_ASTFailureTest')
assert not errata
def test_fail_failtest(self):
"""Failure test should not pass if it failed because the parser is unknown."""
fcases = {}
fcases['berm'] = {}
fcases['berm']['fail'] = self.failure_cases['term']['fail']
errata = grammar_unit(fcases,
grammar_provider(ARITHMETIC_EBNF),
lambda : ARITHMETIC_EBNFTransform,
'REPORT_TestGrammarTest')
assert errata
def test_testing_grammar(self):
parser_fac = grammar_provider(ARITHMETIC_EBNF)
trans_fac = lambda : ARITHMETIC_EBNFTransform
# reset_unit(self.cases)
errata = grammar_unit(self.cases, parser_fac, trans_fac, 'REPORT_TestGrammarTest')
assert errata, "Unknown parser, but no error message!?"
report = get_report(self.cases)
assert report.find('### CST') >= 0
errata = grammar_unit(self.failure_cases, parser_fac, trans_fac, 'REPORT_TestGrammarTest')
assert len(errata) == 3, str(errata)
def test_compileEBNF(self):
parser_src = compileEBNF(ARITHMETIC_EBNF)
assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('DSLGrammar') >= 0
parser_src = compileEBNF(ARITHMETIC_EBNF, branding="CustomDSL")
assert isinstance(parser_src, str), str(type(parser_src))
assert parser_src.find('CustomDSLGrammar') >= 0
factory = grammar_provider(ARITHMETIC_EBNF, branding="TestDSL")
assert callable(factory)
parser = factory()
result = parser("5 + 3 * 4")
assert not is_error(result.error_flag), str(result.errors_sorted)
result = parser("5A + 4B ** 4C")
assert is_error(result.error_flag)
def test_equality2(self):
ebnf = '@literalws = right\nterm = term ("*"|"/") factor | factor\nfactor = /[0-9]+/~'
att = {
"term": [
remove_empty, remove_whitespace, replace_by_single_child,
flatten
],
"factor": [remove_empty, remove_whitespace, reduce_single_child],
"*": [remove_empty, remove_whitespace, replace_by_single_child]
}
parser = grammar_provider(ebnf)()
tree = parser("20 / 4 * 3")
traverse(tree, att)
compare_tree = parse_sxpr(
"(term (term (factor 20) (:Text /) (factor 4)) (:Text *) (factor 3))"
)
assert tree.equals(compare_tree), tree.as_sxpr()
class TestLookahead:
"""
Testing of Expressions with trailing Lookahead-Parser.
"""
EBNF = r"""
document = { category | entry } { LF }
category = { LF } sequence_of_letters { /:/ sequence_of_letters } /:/ §&(LF sequence_of_letters)
entry = { LF } sequence_of_letters !/:/
sequence_of_letters = /[A-Za-z0-9 ]+/
LF = / *\n/
"""
cases = {
"category": {
"match": {
1: """Mountains: big:
K2""", # case 1: matches only with lookahead (but should not fail in a test)
2: """Rivers:""", # case 2: lookahaead failure occurs at end of file and is mandatory. (should not fail as a test)
3: """Mountains: big:""" # same here
},
"fail": {
6: """Mountains: big: """
}
}
}
fail_cases = {
"category": {
"match": {
1: """Mountains: b""", # stop sign ":" is missing
2: """Rivers:
# not allowed""",
2: """Mountains:
K2
Rivers:""" # lookahead only covers K2
},
"fail": {
1: """Mountains: big:
K2"""
}
}
}
grammar_fac = grammar_provider(EBNF)
trans_fac = lambda: partial(traverse, processing_table={"*": [flatten, remove_empty]})
def setup(self):
self.save_dir = os.getcwd()
os.chdir(scriptpath)
def teardown(self):
clean_report('REPORT_TestLookahead')
os.chdir(self.save_dir)
def test_selftest(self):
doc = """
Mountains: big:
Mount Everest
K2
Mountains: medium:
Denali
Alpomayo
Rivers:
Nile
"""
grammar = TestLookahead.grammar_fac()
cst = grammar(doc)
assert not cst.error_flag
def test_unit_lookahead(self):
gr = TestLookahead.grammar_fac()
set_tracer(gr, trace_history)
# Case 1: Lookahead string is part of the test case; parser fails but for the lookahead
result = gr(self.cases['category']['match'][1], 'category')
assert any(e.code in (PARSER_LOOKAHEAD_FAILURE_ONLY,
PARSER_LOOKAHEAD_MATCH_ONLY)
for e in result.errors), str(result.errors)
# Case 2: Lookahead string is not part of the test case; parser matches but for the mandatory continuation
result = gr(self.cases['category']['match'][2], 'category')
assert any(e.code == MANDATORY_CONTINUATION_AT_EOF for e in result.errors)
errata = grammar_unit(self.cases, TestLookahead.grammar_fac, TestLookahead.trans_fac,
'REPORT_TestLookahead')
assert not errata, str(errata)
errata = grammar_unit(self.fail_cases, TestLookahead.grammar_fac, TestLookahead.trans_fac,
'REPORT_TestLookahead')
assert errata
class TestTokenParsing:
ebnf = r"""
@ tokens = BEGIN_INDENT, END_INDENT
@ whitespace = /[ \t]*/
block = { line | indentBlock }+
line = ~/[^\x1b\x1c\x1d\n]*\n/
indentBlock = BEGIN_INDENT block END_INDENT
"""
set_config_value('max_parser_dropouts', 3)
grammar = grammar_provider(ebnf)()
code = lstrip_docstring("""
def func(x, y):
if x > 0: # a comment
if y > 0:
print(x) # another comment
print(y)
""")
tokenized, _ = tokenize_indentation(code)
srcmap = tokenized_to_original_mapping(tokenized, code)
def verify_mapping(self, teststr, orig_text, preprocessed_text, mapping):
mapped_pos = preprocessed_text.find(teststr)
assert mapped_pos >= 0
file_name, file_content, original_pos = mapping(mapped_pos)
# original_pos = source_map(mapped_pos, self.srcmap)
assert orig_text[original_pos:original_pos + len(teststr)] == teststr, \
'"%s" (%i) wrongly mapped onto "%s" (%i)' % \
(teststr, mapped_pos, orig_text[original_pos:original_pos + len(teststr)], original_pos)
def test_strip_tokens(self):
assert self.code == strip_tokens(self.tokenized)
def test_parse_tokenized(self):
cst = self.grammar(self.tokenized)
assert not cst.error_flag
def test_source_mapping_1(self):
mapping = partial(source_map, srcmap=self.srcmap)
self.verify_mapping("def func", self.code, self.tokenized, mapping)
self.verify_mapping("x > 0:", self.code, self.tokenized, mapping)
self.verify_mapping("if y > 0:", self.code, self.tokenized, mapping)
self.verify_mapping("print(x)", self.code, self.tokenized, mapping)
self.verify_mapping("print(y)", self.code, self.tokenized, mapping)
def test_source_mapping_2(self):
previous_index = 0
L = len(self.code)
for mapped_index in range(len(self.tokenized)):
_, _, index = source_map(mapped_index, self.srcmap)
assert previous_index <= index <= L, \
"%i <= %i <= %i violated" % (previous_index, index, L)
previous_index = index
def test_non_token_preprocessor(self):
_, tokenized, mapping, _ = preprocess_comments(self.code, 'no_uri')
self.verify_mapping("def func", self.code, tokenized, mapping)
self.verify_mapping("x > 0:", self.code, tokenized, mapping)
self.verify_mapping("if y > 0:", self.code, tokenized, mapping)
self.verify_mapping("print(x)", self.code, tokenized, mapping)
self.verify_mapping("print(y)", self.code, tokenized, mapping)
def test_chained_preprocessors(self):
pchain = chain_preprocessors(preprocess_comments,
preprocess_indentation)
_, tokenized, mapping, _ = pchain(self.code, 'no_uri')
self.verify_mapping("def func", self.code, tokenized, mapping)
self.verify_mapping("x > 0:", self.code, tokenized, mapping)
self.verify_mapping("if y > 0:", self.code, tokenized, mapping)
self.verify_mapping("print(x)", self.code, tokenized, mapping)
self.verify_mapping("print(y)", self.code, tokenized, mapping)
def test_error_position(self):
orig_src = self.code.replace('#', '\x1b')
prepr = chain_preprocessors(preprocess_comments,
preprocess_indentation)
self.grammar.max_parser_dropouts__ = 3
result, messages, syntaxtree = compile_source(orig_src, prepr,
self.grammar,
lambda i: i, lambda i: i)
for err in messages:
if self.code[err.orig_pos] == "#":
break
else:
assert False, "wrong error positions"