def grammar() -> Grammar:
grammar = Grammar()
whitespace_id = grammar.add_pattern(grammar.add_token('Whitespace'),
r'\s+')
grammar.add_trivia(whitespace_id)
grammar.add_pattern(grammar.add_token('Number'), r'[0-9]+')
grammar.add_pattern(grammar.add_token('Name'), r'[a-zA-Z_][a-zA-Z0-9]+')
grammar.add_implicit("for")
grammar.add_implicit("while")
grammar.add_implicit("+")
grammar.add_implicit("-")
return grammar
def test_add_pattern():
grammar = Grammar()
token_id = grammar.add_token('Name')
result_id = grammar.add_pattern(token_id, r'[a-zA-Z]*')
assert result_id is token_id, "add_pattern must return token id"
assert len(grammar.patterns) == 1
pattern = grammar.patterns[0]
assert pattern.token_id == token_id
assert pattern.pattern == re.compile(r'[a-zA-Z]*')
assert pattern.priority == PRIORITY_MAX
assert not pattern.is_implicit
def test_extend_grammar():
grammar1 = Grammar()
grammar1.add_pattern(grammar1.add_token('A'), 'a+')
grammar1.add_pattern(grammar1.add_token('B'), 'b+')
grammar1.add_pattern(grammar1.add_token('C'), 'c+')
grammar1.add_parselet('expr', kind=ParseletKind.Pratt)
grammar2 = Grammar()
grammar2.add_pattern(grammar2.add_token('A'), '_a+')
grammar2.add_pattern(grammar2.add_token('B'), '_b+')
grammar2.add_parselet('expr', kind=ParseletKind.Pratt)
result = Grammar()
initial_count = len(result.symbols)
result.extend(grammar1)
result.extend(grammar2)
assert 'A' in result.tokens
assert 'B' in result.tokens
assert 'C' in result.tokens
assert len(result.symbols) == initial_count + 4
assert len(result.parselets) == 1
assert len(result.patterns) == 5
assert {pattern.pattern.pattern
for pattern in result.patterns
} == {'a+', '_a+', 'b+', '_b+', 'c+'}
def grammar() -> Grammar:
grammar = Grammar()
whitespace_id = grammar.add_pattern(grammar.add_token('Whitespace'),
r'\s+')
grammar.add_trivia(whitespace_id)
grammar.add_pattern(grammar.add_token('Name'), r'[a-zA-Z_][a-zA-Z0-9]*')
grammar.add_pattern(grammar.add_token('Number'), r'[0-9]+')
make_implicit = grammar.add_implicit
expr_id = grammar.add_parselet('expr',
kind=ParseletKind.Pratt,
result_type=object)
# expr := value:Number
grammar.add_parser(expr_id, "value:Number",
make_call(lambda value: value.value, object))
# expr := lhs:expr op:'+' rhs:expr
grammar.add_parser(expr_id,
'lhs:expr "**" rhs:expr <899>',
make_call(lambda lhs, rhs: (lhs, '**', rhs), object),
priority=900)
# expr := lhs:expr op:'+' rhs:expr
grammar.add_parser(expr_id,
'lhs:expr "+" rhs:expr <600>',
make_call(lambda lhs, rhs: (lhs, '+', rhs), object),
priority=600)
# expr := lhs:expr op:'-' rhs:expr
grammar.add_parser(expr_id,
'lhs:expr "-" rhs:expr <600>',
make_call(lambda lhs, rhs: (lhs, '-', rhs), object),
priority=600)
# expr := lhs:expr op:'*' rhs:expr
grammar.add_parser(expr_id,
'lhs:expr "*" rhs:expr <700>',
make_call(lambda lhs, rhs: (lhs, '*', rhs), object),
priority=700)
# expr := lhs:expr op:'/' rhs:expr
grammar.add_parser(expr_id,
'lhs:expr "/" rhs:expr <700>',
make_call(lambda lhs, rhs: (lhs, '/', rhs), object),
priority=700)
# expr := op:'-' value:expr
grammar.add_parser(expr_id, '"-" value:expr <800>',
make_call(lambda value: ('-', value), object))
# expr := op:'-' value:expr
grammar.add_parser(expr_id, '"+" value:expr <800>',
make_call(lambda value: ('+', value), object))
# expr := '(' value:expr ')'
grammar.add_parser(expr_id, '"(" value:expr ")"',
make_return_variable('value'))
return grammar
def create_core_grammar() -> Grammar:
""" This function is used for initialize default grammar """
grammar = Grammar()
grammar.add_pattern(grammar.add_token('Comment'), RE_COMMENT)
grammar.add_pattern(grammar.add_token('Whitespace'), RE_WHITESPACE)
grammar.add_pattern(grammar.add_token('Name'), RE_NAME)
grammar.add_pattern(grammar.add_token('NewLine'), RE_NEWLINE)
grammar.add_pattern(grammar.add_token('String'), RE_STRING_SINGLE)
grammar.add_pattern(grammar.add_token('String'), RE_STRING_DOUBLE)
grammar.add_pattern(grammar.add_token('Integer'), RE_NUMBER_BINARY)
grammar.add_pattern(grammar.add_token('Integer'), RE_NUMBER_OCTAL)
grammar.add_pattern(grammar.add_token('Integer'), RE_NUMBER_DECIMAL)
grammar.add_pattern(grammar.add_token('Integer'), RE_NUMBER_HEXADECIMAL)
grammar.add_pattern(grammar.add_token('Float'), RE_FLOAT_POINT)
grammar.add_pattern(grammar.add_token('Float'), RE_FLOAT_EXPONENT)
grammar.add_implicit('(')
grammar.add_implicit(')')
grammar.add_implicit('[')
grammar.add_implicit(']')
grammar.add_implicit('{')
grammar.add_implicit('}')
grammar.add_implicit('<')
grammar.add_implicit('>')
grammar.add_trivia(grammar.tokens['Comment'])
grammar.add_trivia(grammar.tokens['Whitespace'])
grammar.add_brackets(grammar.tokens['('], grammar.tokens[')'])
grammar.add_brackets(grammar.tokens['['], grammar.tokens[']'])
grammar.add_brackets(grammar.tokens['{'], grammar.tokens['}'])
return grammar