def test_topython_generates_code_for_complex_clauses(self):
clauses = parsing.Seq(ParseTreeStub('a', False),
parsing.Seq(ParseTreeStub('b', False)))
res = codegen.to_source(ast.Module(passes.rule_topython(clauses)))
self.assertEqual(res, ("if (not a):\n"
" return False\n"
"if (not b):\n"
" return False"))
def test_09_RepRules(self):
"""
Basic test for Rules
"""
def check_word(parser, test, tutu):
test.assertIn(parser.value(tutu), ('asbga', 'njnj'))
return True
def check_int(parser, test, toto):
test.assertIn(parser.value(toto), ('12121', '89898'))
return True
check_word = mock.Mock(side_effect=check_word)
check_int = mock.Mock(side_effect=check_int)
parser = parsing.Parser()
parser.parsed_stream("asbga 12121 njnj 89898")
parser.rule_nodes['test'] = self
parser.set_hooks({'checkWord': check_word, 'checkInt': check_int})
parser.set_rules({
'main':
parsing.Seq(
parsing.Rep0N(
parsing.Alt(
parsing.Seq(
parsing.Capture('tutu', parsing.Rule('word')),
parsing.Hook('checkWord',
[("test", parsing.Node),
("tutu", parsing.Node)])),
parsing.Rule('int'))), parsing.Rule('Base.eof')),
'word':
parsing.Scope(
parsing.Call(parsing.Parser.push_ignore,
parsing.Parser.ignore_null),
parsing.Call(parsing.Parser.pop_ignore),
parsing.Rep1N(
parsing.Alt(
parsing.Call(parsing.Parser.read_range, 'a', 'z'),
parsing.Call(parsing.Parser.read_range, 'A', 'Z')))),
'int':
parsing.Seq(
parsing.Scope(
parsing.Call(parsing.Parser.push_ignore,
parsing.Parser.ignore_null),
parsing.Call(parsing.Parser.pop_ignore),
parsing.Capture(
'toto',
parsing.Rep1N(
parsing.Call(parsing.Parser.read_range, '0',
'9')))),
parsing.Hook('checkInt', [("test", parsing.Node),
("toto", parsing.Node)]))
})
res = parser.eval_rule('main')
self.assertTrue(res, "failed to parse")
self.assertEqual(2, check_word.call_count)
self.assertEqual(2, check_int.call_count)
def test_07_RepXN(self):
"""
Basic test for repeater operator
"""
parser = parsing.Parser()
parser.parsed_stream("12343 91219****1323 23")
parseTree = parsing.Seq(
parsing.Call(parsing.Parser.begin_tag, 'i1'),
parsing.Parser.read_integer,
parsing.Call(parsing.Parser.end_tag, 'i1'),
parsing.Rep0N(parsing.Call(parsing.Parser.read_char, '*')),
parsing.Call(parsing.Parser.begin_tag, 'i2'),
parsing.Rep1N(parsing.Call(parsing.Parser.read_range, '0', '9')),
parsing.Call(parsing.Parser.end_tag, 'i2'),
parsing.Rep0N(parsing.Call(parsing.Parser.read_char, '*')),
parsing.Call(parsing.Parser.begin_tag, 'i3'),
parsing.Parser.read_integer,
parsing.Call(parsing.Parser.end_tag, 'i3'),
parsing.Call(parsing.Parser.read_eof))
parseTree(parser)
# Warning! skip_ignore is called between each parsing.Seq
self.assertEqual(str(parser.get_tag("i1")), "12343 ",
"failed in captured i1")
self.assertEqual(str(parser.get_tag("i2")), "91219",
"failed in captured i2")
self.assertEqual(str(parser.get_tag("i3")), "1323 ",
"failed in captured i3")
def test_it_calls_skipIgnore_before_each_clause(self):
parser = mock.Mock(spec=parsing.BasicParser)
clause = mock.Mock(return_value=True)
parser.clause = clause
parsing.Seq(clause, clause)(parser)
self.assertEqual(
[mock.call.skip_ignore(), mock.call.clause(parser)] * 2,
parser.mock_calls)
def test_it_calls_all_clauses_in_order_if_each_clause_is_true(self):
parser = mock.Mock(spec=parsing.BasicParser)
clauses = mock.Mock(**{'clause0.return_value': True,
'clause1.return_value': True})
parsing.Seq(clauses.clause0, clauses.clause1)(parser)
self.assertEqual(
[mock.call.clause0(parser), mock.call.clause1(parser)],
clauses.mock_calls)
def test_topython_inline_partially_inlinable_clauses(self):
clauses = parsing.Seq(ParseTreeStub('a', True),
ParseTreeStub('b', True),
ParseTreeStub('c', False))
res = codegen.to_source(ast.Module(passes.rule_topython(clauses)))
self.assertEqual(res, ("if (not (a and b)):\n"
" return False\n"
"if (not c):\n"
" return False"))
def test_08_RepAlt(self):
"""
Basic test for alternatives
"""
parser = parsing.Parser()
parser.parsed_stream("_ad121dwdw ()[]")
parseTree = parsing.Seq(
parsing.Call(parsing.Parser.begin_tag, 'w1'),
parsing.Scope(
begin=parsing.Call(parsing.Parser.push_ignore,
parsing.Parser.ignore_null),
end=parsing.Call(parsing.Parser.pop_ignore),
pt=parsing.Seq(
parsing.Alt(
parsing.Call(parsing.Parser.read_char, '_'),
parsing.Call(parsing.Parser.read_range, 'a', 'z'),
parsing.Call(parsing.Parser.read_range, 'A', 'Z')
),
parsing.Rep0N(
parsing.Alt(
parsing.Call(parsing.Parser.read_char, '_'),
parsing.Call(parsing.Parser.read_range, 'a', 'z'),
parsing.Call(parsing.Parser.read_range, 'A', 'Z'),
parsing.Call(parsing.Parser.read_range,
'0', '9'))))),
parsing.Call(parsing.Parser.end_tag, 'w1'),
parsing.Capture(
'w2',
parsing.Rep1N(
parsing.Alt(
parsing.Call(parsing.Parser.read_char, '('),
parsing.Call(parsing.Parser.read_char, ')'),
parsing.Call(parsing.Parser.read_char, '['),
parsing.Call(parsing.Parser.read_char, ']'),
)
)),
parsing.Call(parsing.Parser.read_eof)
)
parseTree(parser)
# Warning! skip_ignore is called between each parsing.Seq
self.assertEqual(str(parser.get_tag("w1")), "_ad121dwdw ",
"failed in captured w1")
self.assertEqual(str(parser.get_tag("w2")), "()[]",
"failed in captured w2")
def test_19_Until(self):
"""
Basic test for until ->R
"""
parser = parsing.Parser()
parser.parsed_stream("==|=|==tutu")
parseTree = parsing.Seq(
parsing.Until(parsing.Call(parsing.Parser.read_text, '|==')),
parsing.Call(parsing.Parser.read_text, 'tutu'),
)
res = parseTree(parser)
self.assertEqual(res, True, "failed to get the correct final value")
def add_sequences(self, sequences, cla) -> bool:
"""Create a tree.Seq"""
if not hasattr(sequences, 'parser_tree'):
# forward sublevel of sequence as is
sequences.parser_tree = cla.parser_tree
else:
oldnode = sequences
if isinstance(oldnode.parser_tree, parsing.Seq):
oldpt = list(oldnode.parser_tree.ptlist)
else:
oldpt = [oldnode.parser_tree]
oldpt.append(cla.parser_tree)
sequences.parser_tree = parsing.Seq(*tuple(oldpt))
return True
def test_16_Negation(self):
"""
Basic test for negation !R
"""
parser = parsing.Parser()
parser.parsed_stream("==")
parseTree = \
parsing.Seq(parsing.Call(parsing.Parser.read_char, '='),
parsing.Neg(parsing.Call(
parsing.Parser.read_char,
'=')))
res = parseTree(parser)
self.assertEqual(res, False, "failed to get the correct final value")
self.assertEqual(parser._stream._cursor._index, 0,
"failed to get the correct index after a negation")
def test_17_Lookahead(self):
"""
Basic test for lookahead !!R
"""
parser = parsing.Parser()
parser.parsed_stream("==")
parseTree = \
parsing.Seq(parsing.Call(parsing.Parser.read_char, '='),
parsing.LookAhead(parsing.Call(
parsing.Parser.read_char,
'=')),
)
res = parseTree(parser)
self.assertEqual(res, True, "failed to get the correct final value")
self.assertEqual(parser._stream._cursor._index, 1,
"failed to get the correct index after a lookahead")
def test_18_Complement(self):
"""
Basic test for complement ~R
"""
parser = parsing.Parser()
parser.parsed_stream("==")
parseTree = parsing.Seq(
parsing.Call(parsing.Parser.read_char, '='),
parsing.Complement(parsing.Call(parsing.Parser.read_char, '=')))
res = parseTree(parser)
self.assertEqual(res, False, "failed to get the correct final value")
self.assertEqual(parser._stream._cursor._index, 0,
"failed to get the correct index after a lookahead")
parser.parsed_stream("=+")
res = parseTree(parser)
self.assertEqual(res, True, "failed to get the correct final value")
self.assertEqual(parser._stream._cursor._index, 2,
"failed to get the correct index after a lookahead")
def test_topython_generates_code_for_complex_alt(self):
self.maxDiff = None
alt = parsing.Alt(
ParseTreeStub('a', False),
parsing.Seq(
ParseTreeStub('b', False),
parsing.Alt(ParseTreeStub('c', False),
ParseTreeStub('d', False))))
res = codegen.to_source(ast.Module(passes.rule_topython(alt)))
self.assertEqual(res, ("try:\n"
" try:\n"
" if (not a):\n"
" raise AltFalse()\n"
" raise AltTrue()\n"
" except AltFalse:\n"
" pass\n"
" try:\n"
" if (not b):\n"
" raise AltFalse()\n"
" try:\n"
" try:\n"
" if (not c):\n"
" raise AltFalse()\n"
" raise AltTrue()\n"
" except AltFalse:\n"
" pass\n"
" try:\n"
" if (not d):\n"
" raise AltFalse()\n"
" raise AltTrue()\n"
" except AltFalse:\n"
" pass\n"
" raise AltFalse()\n"
" except AltTrue:\n"
" pass\n"
" raise AltTrue()\n"
" except AltFalse:\n"
" pass\n"
" return False\n"
"except AltTrue:\n"
" pass"))
def test_06_CallAndSeq(self):
"""
Basic test for call/clauses
"""
parser = parsing.Parser()
parser.parsed_stream("abc def ghg")
parseTree = parsing.Seq(parsing.Call(parsing.Parser.begin_tag, 'i1'),
parsing.Parser.read_identifier,
parsing.Call(parsing.Parser.end_tag, 'i1'),
parsing.Call(parsing.Parser.begin_tag, 'i2'),
parsing.Parser.read_identifier,
parsing.Call(parsing.Parser.end_tag, 'i2'),
parsing.Call(parsing.Parser.begin_tag, 'i3'),
parsing.Parser.read_identifier,
parsing.Call(parsing.Parser.end_tag, 'i3'))
parseTree(parser)
# Warning! skip_ignore is called between each parsing.Seq
self.assertEqual(str(parser.get_tag("i1")), "abc ",
"failed in captured i1")
self.assertEqual(str(parser.get_tag("i2")), "def ",
"failed in captured i2")
self.assertEqual(str(parser.get_tag("i3")), "ghg",
"failed in captured i3")
def __init__(self, content='', sname=None):
"""
Define the DSL parser.
"""
super().__init__(content, sname)
self.set_rules({
#
# bnf_dsl = [ @ignore("C/C++") bnf_stmts ]
# //todo: bnf_dsl = [ @ignore("C/C++") [bnf_stmts] eof ]
#
'bnf_dsl':
parsing.Seq(
# tree is not already construct but Directive need it
# forward it thru a lambda
parsing.Directive(
ignore.Ignore(), [("C/C++", str)],
lambda parser: self.__class__._rules['bnf_stmts']
(parser)), ),
#
# bnf_stmts = [ [rule : r #add_rules(_, r) ]+ Base.eof ]
# //todo: bnf_stmts = [ [rule : r #add_rules(_, r) ]+]
#
'bnf_stmts':
parsing.Seq(
parsing.Rep1N(
parsing.Seq(
parsing.Capture("r", parsing.Rule('rule')),
parsing.Hook('add_rules', [("_", parsing.Node),
("r", parsing.Node)]))),
parsing.Rule('Base.eof')),
# TODO: add directive hooks / change ns_name by def_rule
#
# rule = [ ns_name : rn '=' '[' alternatives : alts
# #add_rule(_, rn, alts) ']' ]
#
'rule':
parsing.Seq(
parsing.Capture("rn", parsing.Rule('ns_name')),
parsing.Alt(parsing.Char("="), parsing.Error("Expected '='")),
parsing.Alt(parsing.Char("["), parsing.Error("Expected '['")),
parsing.Capture("alts", parsing.Rule('alternatives')),
parsing.Hook('add_rule', [("_", parsing.Node),
("rn", parsing.Node),
("alts", parsing.Node)]),
parsing.Alt(
#parsing.Call(parsing.Parser.read_char, ']'),
parsing.Char(']'),
parsing.Error("Expected ']'"))),
#
# alternatives =
# [
# sequences : alt #add_alt(_, alt)
# ['|' sequences : alt #add_alt(_, alt) ]*
# ]
#
'alternatives':
parsing.Seq(
parsing.Capture('alt', parsing.Rule('sequences')),
parsing.Hook('add_alt', [("_", parsing.Node),
("alt", parsing.Node)]),
parsing.Rep0N(
parsing.Seq(
parsing.Char('|'),
parsing.Capture('alt', parsing.Rule('sequences')),
parsing.Hook('add_alt', [("_", parsing.Node),
("alt", parsing.Node)])))),
#
# sequences = [ [ sequence : cla #add_sequences(_, cla) ]+ ]
#
'sequences':
parsing.Rep1N(
parsing.Seq(
parsing.Capture('cla', parsing.Rule('sequence')),
parsing.Hook('add_sequences', [("_", parsing.Node),
("cla", parsing.Node)]))),
#
# sequence = [
# [
# [ '~' | "!!" | '!' | "->" ]?: mod
# [ ns_name : rid #add_ruleclause_name(_, rid)
# | Base.string : txt #add_text(_, txt)
# | Base.char : begin ".." Base.char : end
# #add_range(_, begin, end)
# | Base.char : c #add_char(_, c)
# | '[' alternatives : subsequence ']'
# #add_subsequence(_, subsequence)
# ] #add_mod(_, mod)
# [ repeat : rpt #add_rpt(_, mod, rpt) ]?
# | hook : h #add_hook(_, h)
# | directive2 : d sequences : s #add_directive2(_, d, s)
# | directive : d sequences : s #add_directive(_, d, s)
# ]
# [
# ":>" Base.id : bind #add_bind(_, bind)
# | ':' Base.id : cpt #add_capture(_, cpt)
# ]?
# ]
#
'sequence':
parsing.Seq(
parsing.Alt(
parsing.Seq(
parsing.Capture(
'mod',
parsing.RepOptional(
parsing.Alt(parsing.Char('~'),
parsing.Text('!!'),
parsing.Char('!'),
parsing.Text('->')))),
parsing.Alt(
parsing.Seq(
parsing.Capture('rid',
parsing.Rule('ns_name')),
parsing.Hook('add_ruleclause_name',
[("_", parsing.Node),
("rid", parsing.Node)])),
parsing.Seq(
parsing.Capture('txt',
parsing.Rule('Base.string')),
parsing.Hook('add_text',
[("_", parsing.Node),
("txt", parsing.Node)])),
parsing.Seq(
parsing.Capture('begin',
parsing.Rule('Base.char')),
parsing.Text(".."),
parsing.Capture('end',
parsing.Rule('Base.char')),
parsing.Hook('add_range',
[("_", parsing.Node),
("begin", parsing.Node),
("end", parsing.Node)])),
parsing.Seq(
parsing.Capture('c',
parsing.Rule('Base.char')),
parsing.Hook('add_char',
[("_", parsing.Node),
("c", parsing.Node)])),
parsing.Seq(
parsing.Char('['),
parsing.Capture(
'subsequence',
parsing.Alt(
parsing.Rule('alternatives'),
parsing.Error("Expected sequences"))),
parsing.Alt(parsing.Char(']'),
parsing.Error("Expected ']'")),
parsing.Hook('add_subsequence',
[("_", parsing.Node),
("subsequence", parsing.Node)]),
)),
parsing.Hook('add_mod', [("_", parsing.Node),
("mod", parsing.Node)]),
parsing.RepOptional(
parsing.Seq(
parsing.Capture('rpt', parsing.Rule('repeat')),
parsing.Hook('add_rpt',
[("_", parsing.Node),
("mod", parsing.Node),
("rpt", parsing.Node)]))),
),
parsing.Seq(
parsing.Capture('h', parsing.Rule('hook')),
parsing.Hook('add_hook', [('_', parsing.Node),
('h', parsing.Node)])),
parsing.Seq(
parsing.Capture('d', parsing.Rule('directive2')),
parsing.Capture('s', parsing.Rule('sequences')),
parsing.Hook('add_directive2', [('_', parsing.Node),
('d', parsing.Node),
('s', parsing.Node)])),
parsing.Seq(
parsing.Capture('d', parsing.Rule('directive')),
parsing.Capture('s', parsing.Rule('sequences')),
parsing.Hook('add_directive', [('_', parsing.Node),
('d', parsing.Node),
('s', parsing.Node)]))),
parsing.RepOptional(
parsing.Alt(
parsing.Seq(
parsing.Text(':>'),
parsing.Capture('bind', parsing.Rule('Base.id')),
parsing.Hook('add_bind',
[('_', parsing.Node),
('bind', parsing.Node)])),
parsing.Seq(
parsing.Char(':'),
parsing.Capture('cpt', parsing.Rule('Base.id')),
parsing.Hook('add_capture',
[('_', parsing.Node),
('cpt', parsing.Node)]))))),
# ns_name = [ [@ignore("null") [ Base.id ['.' Base.id]* ]]: rid ]
#
'ns_name':
parsing.Capture(
'rid',
parsing.Scope(
parsing.Call(parsing.Parser.push_ignore,
parsing.Parser.ignore_null),
parsing.Call(parsing.Parser.pop_ignore),
parsing.Seq(
parsing.Rule('Base.id'),
parsing.Rep0N(
parsing.Seq(
parsing.Char('.'),
parsing.Alt(
parsing.Rule('Base.id'),
parsing.Error(
"Expected identifier after '.'"))))))),
#
# repeat = [ '?' #add_optional(_)
# | '*' #add_0N(_)
# | '+' #add_1N(_)
# ]
#
'repeat':
parsing.Alt(
parsing.Seq(
parsing.Char('?'),
parsing.Hook('add_optional', [("_", parsing.Node)])),
parsing.Seq(parsing.Char('*'),
parsing.Hook('add_0N', [("_", parsing.Node)])),
parsing.Seq(parsing.Char('+'),
parsing.Hook('add_1N', [("_", parsing.Node)])),
),
#
# hook = [ '#' ns_name : n #hook_name(_, n)
# ['(' param : p #hook_param(_, p)
# [',' param : p #hook_param(_, p)]*
# ')']?
# ]
#
'hook':
parsing.Seq(
parsing.Char('#'),
parsing.Capture('n', parsing.Rule('ns_name')),
parsing.Hook('hook_name', [('_', parsing.Node),
('n', parsing.Node)]),
parsing.RepOptional(
parsing.Seq(
parsing.Char('('),
parsing.Capture(
'p',
parsing.Alt(parsing.Rule('param'),
parsing.Error("Expected parameter"))),
parsing.Hook('hook_param', [('_', parsing.Node),
('p', parsing.Node)]),
parsing.Rep0N(
parsing.Seq(
parsing.Char(','),
parsing.Capture(
'p',
parsing.Alt(
parsing.Rule('param'),
parsing.Error("Expected parameter"))),
parsing.Hook('hook_param',
[('_', parsing.Node),
('p', parsing.Node)]))),
parsing.Alt(parsing.Char(')'),
parsing.Error("Expected ')'")))),
),
#
# directive2 = [ '$' ns_name : n #hook_name(_, n)
# ['(' param : p #hook_param(_, p)
# [',' param : p #hook_param(_, p)]*
# ')']?
# ]
'directive2':
parsing.Seq(
parsing.Char('$'),
parsing.Capture('n', parsing.Rule('ns_name')),
parsing.Hook('hook_name', [('_', parsing.Node),
('n', parsing.Node)]),
parsing.RepOptional(
parsing.Seq(
parsing.Char('('),
parsing.Capture(
'p',
parsing.Alt(parsing.Rule('param'),
parsing.Error("Expected parameter"))),
parsing.Hook('hook_param', [('_', parsing.Node),
('p', parsing.Node)]),
parsing.Rep0N(
parsing.Seq(
parsing.Char(','),
parsing.Capture(
'p',
parsing.Alt(
parsing.Rule('param'),
parsing.Error("Expected parameter"))),
parsing.Hook('hook_param',
[('_', parsing.Node),
('p', parsing.Node)]),
)),
parsing.Alt(parsing.Char(')'),
parsing.Error("Expected ')'")))),
),
#
# directive = [ '@' ns_name : n #hook_name(_, n)
# ['(' param : p #hook_param(_, p)
# [',' param : p #hook_param(_, p)]*
# ')']?
# ]
'directive':
parsing.Seq(
parsing.Char('@'),
parsing.Capture('n', parsing.Rule('ns_name')),
parsing.Hook('hook_name', [('_', parsing.Node),
('n', parsing.Node)]),
parsing.RepOptional(
parsing.Seq(
parsing.Char('('),
parsing.Capture(
'p',
parsing.Alt(parsing.Rule('param'),
parsing.Error("Expected parameter"))),
parsing.Hook('hook_param', [('_', parsing.Node),
('p', parsing.Node)]),
parsing.Rep0N(
parsing.Seq(
parsing.Char(','),
parsing.Capture(
'p',
parsing.Alt(
parsing.Rule('param'),
parsing.Error("Expected parameter"))),
parsing.Hook('hook_param',
[('_', parsing.Node),
('p', parsing.Node)]),
)),
parsing.Alt(parsing.Char(')'),
parsing.Error("Expected ')'")))),
),
#
# param = [ Base.num :n #param_num(_, n)
# | Base.string : s #param_str(_, s)
# | Base.char : c #param_char(_, c)
# | ns_name : i #param_id(_, i)
# ]
#
'param':
parsing.Alt(
parsing.Seq(
parsing.Capture('n', parsing.Rule('Base.num')),
parsing.Hook('param_num', [('_', parsing.Node),
('n', parsing.Node)])),
parsing.Seq(
parsing.Capture('s', parsing.Rule('Base.string')),
parsing.Hook('param_str', [('_', parsing.Node),
('s', parsing.Node)])),
parsing.Seq(
parsing.Capture('c', parsing.Rule('Base.char')),
parsing.Hook('param_char', [('_', parsing.Node),
('c', parsing.Node)])),
parsing.Seq(
parsing.Capture('i', parsing.Rule('ns_name')),
parsing.Hook('param_id', [('_', parsing.Node),
('i', parsing.Node)])),
),
})
def test_it_is_true_if_all_clauses_are_true(self):
parser = mock.Mock(spec=parsing.BasicParser)
clause = mock.Mock(return_value=True)
clauses = parsing.Seq(clause, clause)
self.assertTrue(clauses(parser))
def test_is_is_false_if_any_clause_is_false(self):
parser = mock.Mock(spec=parsing.BasicParser)
clause0 = mock.Mock(return_value=True)
clause1 = mock.Mock(return_value=False)
clauses = parsing.Seq(clause0, clause1)
self.assertFalse(clauses(parser))
def test_it_raises_typeerror_with_no_clause(self):
with self.assertRaises(TypeError):
parsing.Seq()
def test_topython_inline_inlinable_clauses(self):
clauses = parsing.Seq(ParseTreeStub('a', True),
ParseTreeStub('b', True))
res = codegen.to_source(ast.Module(passes.rule_topython(clauses)))
self.assertEqual(res, "(a and b)")
def test_it_stops_calling_clauses_if_a_clause_is_false(self):
parser = mock.Mock(spec=parsing.BasicParser)
clauses = mock.Mock(**{'clause0.return_value': False,
'clause1.return_value': True})
parsing.Seq(clauses.clause0, clauses.clause1)(parser)
self.assertEqual([mock.call.clause0(parser)], clauses.mock_calls)
