def setup_method(self, method):
self.parser = Parser()
self.tok1 = self.parser.Token_n("t1", 'foo')
self.tok2 = self.parser.Token_n("t2", 'bar')
self.tok3 = self.parser.Token_n("t3", 'foobar')
self.tokens = [self.tok1, self.tok2, self.tok3]
self.parser.build_first_sets()
def build_parser(gramfile, parser=None):
"""reads a (EBNF) grammar definition and builds a parser for it"""
if parser is None:
parser = Parser()
setup_tokens(parser)
# XXX: clean up object dependencies
from pypy.rlib.streamio import open_file_as_stream
stream = open_file_as_stream(gramfile)
grammardef = stream.readall()
stream.close()
assert isinstance(grammardef, str)
source = GrammarSource(GRAMMAR_GRAMMAR, grammardef)
builder = EBNFBuilder(GRAMMAR_GRAMMAR, dest_parser=parser)
GRAMMAR_GRAMMAR.root_rules['grammar'].match(source, builder)
builder.resolve_rules()
parser.build_first_sets()
parser.keywords = builder.keywords
return parser
def build_parser(gramfile, parser=None):
"""reads a (EBNF) grammar definition and builds a parser for it"""
if parser is None:
parser = Parser()
setup_tokens(parser)
# XXX: clean up object dependencies
from pypy.rlib.streamio import open_file_as_stream
stream = open_file_as_stream(gramfile)
grammardef = stream.readall()
stream.close()
assert isinstance(grammardef, str)
source = GrammarSource(GRAMMAR_GRAMMAR, grammardef)
builder = EBNFBuilder(GRAMMAR_GRAMMAR, dest_parser=parser)
GRAMMAR_GRAMMAR.root_rules['grammar'].match(source, builder)
builder.resolve_rules()
parser.build_first_sets()
parser.keywords = builder.keywords
return parser
def setup_method(self, method):
p = self.parser = Parser()
self.LOW = p.Token_n( 'LOW', 'low')
self.CAP = p.Token_n( 'CAP' ,'cap')
self.A = p.Alternative_n( 'R_A', [])
k1 = p.KleeneStar_n( 'R_k1', 0, rule=self.LOW)
k2 = p.KleeneStar_n( 'R_k2', 0, rule=self.CAP)
self.B = p.Sequence_n( 'R_B', [k1, self.A])
self.C = p.Sequence_n( 'R_C', [k2, self.A])
self.A.args = [self.B, self.C]
p.build_first_sets()
def test_symbols():
p = Parser()
x1 = p.add_symbol('sym')
x2 = p.add_token('tok')
x3 = p.add_anon_symbol(':sym')
x4 = p.add_anon_symbol(':sym1')
# test basic numbering assumption
# symbols and tokens are attributed sequentially
# using the same counter
assert x2 == x1 + 1
# anon symbols have negative value
assert x3 != x2 + 1
assert x4 == x3 - 1
assert x3 < 0
y1 = p.add_symbol('sym')
assert y1 == x1
y2 = p.add_token('tok')
assert y2 == x2
y3 = p.add_symbol(':sym')
assert y3 == x3
y4 = p.add_symbol(':sym1')
assert y4 == x4
def test_load():
d = { 5 : 'sym1',
6 : 'sym2',
9 : 'sym3',
}
p = Parser()
p.load_symbols( d )
v = p.add_symbol('sym4')
# check that we avoid numbering conflicts
assert v>9
v = p.add_symbol( 'sym1' )
assert v == 5
v = p.add_symbol( 'sym2' )
assert v == 6
v = p.add_symbol( 'sym3' )
assert v == 9
class TestLookAheadBasics:
def setup_method(self, method):
self.parser = Parser()
self.tok1 = self.parser.Token_n("t1", 'foo')
self.tok2 = self.parser.Token_n("t2", 'bar')
self.tok3 = self.parser.Token_n("t3", 'foobar')
self.tokens = [self.tok1, self.tok2, self.tok3]
self.parser.build_first_sets()
def test_basic_token(self):
assert self.tok1.first_set == [self.tok1]
def test_basic_alternative(self):
alt = self.parser.Alternative_n("a1t", self.tokens)
self.parser.build_first_sets()
assert alt.first_set == self.tokens
def test_basic_sequence(self):
seq = self.parser.Sequence_n("seq", self.tokens)
self.parser.build_first_sets()
assert seq.first_set == [self.tokens[0]]
def test_basic_kleenstar(self):
tok1, tok2, tok3 = self.tokens
kstar1 = self.parser.KleeneStar_n("k", 1, 3, tok1)
kstar2 = self.parser.KleeneStar_n("k2", 0, 3, tok1)
self.parser.build_first_sets()
assert kstar1.first_set == [tok1]
assert kstar2.first_set == [tok1, self.parser.EmptyToken]
def test_maybe_empty_sequence(self):
"""S -> tok1{0,2} tok2{0,2}
==> S.first_set = [tok1, tok2, EmptyToken]
"""
tok1, tok2, tok3 = self.tokens
k1 = self.parser.KleeneStar_n( "k1", 0, 2, tok1)
k2 = self.parser.KleeneStar_n("k2", 0, 2, tok2)
seq = self.parser.Sequence_n( "seq", [k1, k2])
self.parser.build_first_sets()
assert seq.first_set == [tok1, tok2, self.parser.EmptyToken]
def test_not_empty_sequence(self):
"""S -> tok1{0,2} tok2{1,2}
==> S.first_set = [tok1, tok2]
"""
tok1, tok2, tok3 = self.tokens
k1 = self.parser.KleeneStar_n("k1", 0, 2, tok1)
k2 = self.parser.KleeneStar_n("k2", 1, 2, tok2)
seq = self.parser.Sequence_n("seq", [k1, k2])
self.parser.build_first_sets()
assert seq.first_set == [tok1, tok2]
def test_token_comparison(self):
tok1 = self.parser.Token_n( "tok1", "foo" )
tok1b = self.parser.Token_n( "tok1", "foo" )
tok2 = self.parser.Token_n( "tok2", "foo" )
tok3 = self.parser.Token_n( "tok2", None )
assert tok1 == tok1b
assert tok1 != tok2
assert tok2 != tok3
class TestLookAheadBasics:
def setup_method(self, method):
self.parser = Parser()
self.tok1 = self.parser.Token_n("t1", 'foo')
self.tok2 = self.parser.Token_n("t2", 'bar')
self.tok3 = self.parser.Token_n("t3", 'foobar')
self.tokens = [self.tok1, self.tok2, self.tok3]
self.parser.build_first_sets()
def test_basic_token(self):
assert self.tok1.get_first_set() == {self.tok1: None}
def test_basic_alternative(self):
alt = self.parser.Alternative_n("a1t", self.tokens)
self.parser.build_first_sets()
assert alt.get_first_set() == dict.fromkeys(self.tokens)
def test_basic_sequence(self):
seq = self.parser.Sequence_n("seq", self.tokens)
self.parser.build_first_sets()
assert seq.get_first_set() == {self.tokens[0]: None}
def test_basic_kleenstar(self):
tok1, tok2, tok3 = self.tokens
kstar1 = self.parser.KleeneStar_n("k", 1, 3, tok1)
kstar2 = self.parser.KleeneStar_n("k2", 0, 3, tok1)
self.parser.build_first_sets()
assert kstar1.get_first_set() == {tok1: None}
assert kstar2.get_first_set() == {tok1: None,
self.parser.EmptyToken: None}
def test_maybe_empty_sequence(self):
"""S -> tok1{0,2} tok2{0,2}
==> S.first_set = {tok1, tok2, EmptyToken}
"""
tok1, tok2, tok3 = self.tokens
k1 = self.parser.KleeneStar_n( "k1", 0, 2, tok1)
k2 = self.parser.KleeneStar_n("k2", 0, 2, tok2)
seq = self.parser.Sequence_n( "seq", [k1, k2])
self.parser.build_first_sets()
assert seq.get_first_set() == {tok1: None,
tok2: None,
self.parser.EmptyToken: None}
def test_not_empty_sequence(self):
"""S -> tok1{0,2} tok2{1,2}
==> S.first_set = {tok1, tok2}
"""
tok1, tok2, tok3 = self.tokens
k1 = self.parser.KleeneStar_n("k1", 0, 2, tok1)
k2 = self.parser.KleeneStar_n("k2", 1, 2, tok2)
seq = self.parser.Sequence_n("seq", [k1, k2])
self.parser.build_first_sets()
assert seq.get_first_set() == {tok1: None, tok2: None}
def test_token_comparison(self):
tok1 = self.parser.Token_n( "tok1", "foo" )
tok1b = self.parser.Token_n( "tok1", "foo" )
tok2 = self.parser.Token_n( "tok2", "foo" )
tok3 = self.parser.Token_n( "tok2", None )
assert tok1.eq(tok1b)
assert not tok1.eq(tok2)
assert not tok2.eq(tok3)
