def test_make_optional(): grammar = Grammar() name_id = grammar.add_token('Name') comb = make_optional(name_id) assert isinstance(comb, OptionalCombinator) assert isinstance(comb.combinator, TokenCombinator) assert comb.result_type == Optional[SyntaxToken]
def test_make_token(): grammar = Grammar() name_id = grammar.add_token('Name') comb = make_token(name_id) assert isinstance(comb, TokenCombinator) assert comb.token_id == name_id assert comb.result_type == SyntaxToken assert comb.variables == {}
def test_extend_packrat_grammar(): grammar1 = Grammar() grammar1.add_token('Number') grammar1.add_token('String') expr_id = grammar1.add_parselet('expr', result_type=object) grammar1.add_parser(expr_id, 'Number') grammar1.add_parser(expr_id, 'String') grammar2 = Grammar() grammar2.add_token('Number') grammar2.add_token('String') expr_id = grammar2.add_parselet('expr', result_type=object) grammar2.add_parser(expr_id, 'Number') grammar2.add_parser(expr_id, 'String') result = Grammar.merge(grammar1, grammar2) expr_id = result.parselets['expr'] assert expr_id in result.tables assert len(cast(PackratTable, result.tables[expr_id]).parselets) == 4
def test_add_token(): grammar = Grammar() token_id = grammar.add_token('Name') assert 'Name' in grammar.tokens assert len(grammar.patterns) == 0 assert token_id == token_id assert token_id.name == 'Name' assert token_id.description == 'name' assert not token_id.is_implicit
def test_make_sequence(): grammar = Grammar() name_id = grammar.add_token('Name') expr_id = grammar.add_parselet('expr') comb = make_sequence(name_id, expr_id) assert isinstance(comb, SequenceCombinator) assert len(comb) == 2 assert isinstance(comb[0], TokenCombinator) assert isinstance(comb[1], ParseletCombinator) assert comb.result_type == SyntaxNode
def test_add_pattern(): grammar = Grammar() token_id = grammar.add_token('Name') result_id = grammar.add_pattern(token_id, r'[a-zA-Z]*') assert result_id is token_id, "add_pattern must return token id" assert len(grammar.patterns) == 1 pattern = grammar.patterns[0] assert pattern.token_id == token_id assert pattern.pattern == re.compile(r'[a-zA-Z]*') assert pattern.priority == PRIORITY_MAX assert not pattern.is_implicit
def test_add_pratt_parser(): grammar = Grammar() expr_id = grammar.add_parselet('expr', kind=ParseletKind.Pratt, result_type=SyntaxToken) integer_id = grammar.add_token('Integer') string_id = grammar.add_token('String') plus_id = grammar.add_implicit('+') star_id = grammar.add_implicit('*') table = cast(PrattTable, grammar.tables[expr_id]) assert table.prefix_tokens == set() assert grammar.add_parser(expr_id, integer_id) assert integer_id in table.prefix_tokens, "Cleanup of pratt table prefix tokens is not worked" assert grammar.add_parser(expr_id, make_named('value', string_id)) assert string_id in table.prefix_tokens, "Cleanup of pratt table prefix tokens is not worked" assert grammar.add_parser(expr_id, make_sequence(expr_id, plus_id, expr_id)) assert grammar.add_parser( expr_id, make_sequence(make_named('lhs', expr_id), make_named('op', star_id), expr_id))
def test_flat_sequence(): grammar = Grammar() name_id = grammar.add_token('Name') expr_id = grammar.add_parselet('expr') combinators = tuple( flat_sequence(TokenCombinator(name_id), ParseletCombinator(expr_id), SequenceCombinator(( TokenCombinator(name_id), ParseletCombinator(expr_id), )), kind=SequenceCombinator)) assert len(combinators) == 4 assert isinstance(combinators[0], TokenCombinator) assert isinstance(combinators[1], ParseletCombinator) assert isinstance(combinators[2], TokenCombinator) assert isinstance(combinators[3], ParseletCombinator)
def test_flat_combinator(): grammar = Grammar() name_id = grammar.add_token('Name') expr_id = grammar.add_parselet('expr') # convert token id to token combinator comb = flat_combinator(name_id) assert isinstance(comb, TokenCombinator) assert comb.token_id is name_id # convert parselet id to parselet combinator comb = flat_combinator(expr_id) assert isinstance(comb, ParseletCombinator) assert comb.parser_id is expr_id assert comb.priority is None # don't convert combinator comb = TokenCombinator(name_id) result = flat_combinator(comb) assert comb is result
def grammar() -> Grammar: grammar = Grammar() whitespace_id = grammar.add_pattern(grammar.add_token('Whitespace'), r'\s+') grammar.add_trivia(whitespace_id) grammar.add_pattern(grammar.add_token('Name'), r'[a-zA-Z_][a-zA-Z0-9]*') grammar.add_pattern(grammar.add_token('Number'), r'[0-9]+') make_implicit = grammar.add_implicit expr_id = grammar.add_parselet('expr', kind=ParseletKind.Pratt, result_type=object) # expr := value:Number grammar.add_parser(expr_id, "value:Number", make_call(lambda value: value.value, object)) # expr := lhs:expr op:'+' rhs:expr grammar.add_parser(expr_id, 'lhs:expr "**" rhs:expr <899>', make_call(lambda lhs, rhs: (lhs, '**', rhs), object), priority=900) # expr := lhs:expr op:'+' rhs:expr grammar.add_parser(expr_id, 'lhs:expr "+" rhs:expr <600>', make_call(lambda lhs, rhs: (lhs, '+', rhs), object), priority=600) # expr := lhs:expr op:'-' rhs:expr grammar.add_parser(expr_id, 'lhs:expr "-" rhs:expr <600>', make_call(lambda lhs, rhs: (lhs, '-', rhs), object), priority=600) # expr := lhs:expr op:'*' rhs:expr grammar.add_parser(expr_id, 'lhs:expr "*" rhs:expr <700>', make_call(lambda lhs, rhs: (lhs, '*', rhs), object), priority=700) # expr := lhs:expr op:'/' rhs:expr grammar.add_parser(expr_id, 'lhs:expr "/" rhs:expr <700>', make_call(lambda lhs, rhs: (lhs, '/', rhs), object), priority=700) # expr := op:'-' value:expr grammar.add_parser(expr_id, '"-" value:expr <800>', make_call(lambda value: ('-', value), object)) # expr := op:'-' value:expr grammar.add_parser(expr_id, '"+" value:expr <800>', make_call(lambda value: ('+', value), object)) # expr := '(' value:expr ')' grammar.add_parser(expr_id, '"(" value:expr ")"', make_return_variable('value')) return grammar
def test_make_sequence_with_single_element(): grammar = Grammar() name_id = grammar.add_token('Name') comb = make_sequence(name_id) assert isinstance(comb, TokenCombinator)
def test_add_idempotent_trivia(): grammar = Grammar() token_id = grammar.add_token('Whitespace') for _ in range(3): grammar.add_trivia(token_id) assert grammar.trivia == {token_id}
def test_add_trivia(): grammar = Grammar() token_id = grammar.add_token('Whitespace') assert grammar.trivia == set() grammar.add_trivia(token_id) assert grammar.trivia == {token_id}
def test_add_idempotent_token(): grammar = Grammar() t1 = grammar.add_token('Name') t2 = grammar.add_token('Name') assert t1 is t2 and t1 == t2
def test_add_fast_parser(): grammar = Grammar() grammar.add_token('Number') grammar.add_parser('expr', 'Number')