def __init__(self, module, builder, printf, scanf, definations = {}):
self.pg = ParserGenerator(
# Tokens that can be accepted by our parser
['NUMBER', 'WRITE', 'WRITELN', 'OPEN_PAREN', 'CLOSE_PAREN',
'SEMI_COLON', 'SUM', 'SUB','MUL','DIV','MOD', 'VAR', 'ASSIGN',
'AND', 'OR', 'NOT', 'TRUE', 'FALSE',
'EQUALS', 'LESS', 'GREATER', 'LESS_EQ', 'GREAT_EQ', 'NOT_EQUALS',
'COMMA', 'STRING', 'IF', 'ELSE', 'OPEN_CURLY', 'CLOSE_CURLY',
'NOPS','FUNCTION', 'RETURN', 'FOR', 'INPUT', 'WHILE'
],
## Defining the precedence of operators in language
precedence = [
('left', ['SUM', 'SUB']),
('left', ['MUL', 'DIV']),
('left', ['MOD'])
]
)
## Setting the module, builder and printf system call reference
self.module = module
self.builder = builder
self.printf = printf
self.scanf = scanf
## Initializing the defaults constructs for our language
## Like a global string called True, False etc.
initialize(builder, module, definations)
self.constants = {}
self.constants['false'] = self.builder.bitcast(globalFalse, globalVoidPtr)
self.constants['true'] = self.builder.bitcast(globalTrue, globalVoidPtr)
self.constants['int'] = self.builder.bitcast(globalInt, globalVoidPtr)
def __init__(self, syntax=False):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
[
'STRING', 'INTEGER', 'FLOAT', 'BOOLEAN', 'PI', 'E', 'PRINT',
'ABSOLUTE', 'SIN', 'COS', 'TAN', 'POWER', 'CONSOLE_INPUT', '(',
')', ';', ',', '{', '}', 'LET', 'AND', 'OR', 'NOT', 'IF',
'ELSE', '=', '==', '!=', '>=', '>', '<', '<=', 'SUM', 'SUB',
'MUL', 'DIV', 'IDENTIFIER', 'FUNCTION'
],
# A list of precedence rules with ascending precedence, to
# disambiguate ambiguous production rules.
precedence=(('left', ['FUNCTION']), ('left', ['LET']),
('left', ['=']), ('left', ['IF', 'ELSE', ';']),
('left', ['AND', 'OR']), ('left', ['NOT']),
('left', ['==', '!=', '>=', '>', '<',
'<=']), ('left', ['SUM',
'SUB']), ('left',
['MUL', 'DIV']),
('left',
['STRING', 'INTEGER', 'FLOAT', 'BOOLEAN', 'PI',
'E'])))
self.syntax = syntax
self.parse()
pass # End Parser's constructor !
def __init__(self):
# The list of tokens from the lexer file
self.pg = ParserGenerator([token for token in operators.keys()],
precedence=[
("left", ["ADD", "SUBTRACT"]),
("left", ["MULTIPLY", "DIVIDE"]),
])
def test_empty_production(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : values")
def main(p):
return p[0]
@pg.production("values : VALUE values")
def values_value(p):
return [p[0]] + p[1]
@pg.production("values :")
def values_empty(p):
return []
parser = pg.build()
assert parser.lr_table.lr_action == [
{
"$end": -3,
"VALUE": 3
},
{
"$end": 0
},
{
"$end": -1
},
{
"$end": -3,
"VALUE": 3
},
{
"$end": -2
},
]
def test_basic_parser(self):
pg = ParserGenerator(["NUMBER", "PLUS"])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
def expr_op(p):
return BoxInt(p[0].getint() + p[2].getint())
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
with self.assert_warns(
ParserGeneratorWarning, "1 shift/reduce conflict"
):
parser = pg.build()
def f(n):
return parser.parse(iter([
Token("NUMBER", str(n)),
Token("PLUS", "+"),
Token("NUMBER", str(n))
])).getint()
assert self.run(f, [12]) == 24
def test_precedence(self):
pg = ParserGenerator(["NUMBER", "PLUS", "TIMES"], precedence=[
("left", ["PLUS"]),
("left", ["TIMES"]),
])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
@pg.production("expr : expr TIMES expr")
def expr_binop(p):
return BoxInt({
"+": operator.add,
"*": operator.mul
}[p[1].getstr()](p[0].getint(), p[2].getint()))
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
parser = pg.build()
assert parser.parse(iter([
Token("NUMBER", "3"),
Token("TIMES", "*"),
Token("NUMBER", "4"),
Token("PLUS", "+"),
Token("NUMBER", "5")
])) == BoxInt(17)
def test_arithmetic(self):
lg = LexerGenerator()
lg.add("NUMBER", r"\d+")
lg.add("PLUS", r"\+")
lg.add("TIMES", r"\*")
pg = ParserGenerator(["NUMBER", "PLUS", "TIMES"], precedence=[
("left", ["PLUS"]),
("left", ["TIMES"]),
])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
@pg.production("expr : expr TIMES expr")
def expr_binop(p):
return BoxInt({
"+": operator.add,
"*": operator.mul
}[p[1].getstr()](p[0].getint(), p[2].getint()))
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
lexer = lg.build()
parser = pg.build()
assert parser.parse(lexer.lex("3*4+5"))
def test_null_production(self):
pg = ParserGenerator(["VALUE", "SPACE"])
@pg.production("main : values")
def main(p):
return p[0]
@pg.production("values : none")
def values_empty(p):
return []
@pg.production("values : VALUE")
def values_value(p):
return [p[0].getstr()]
@pg.production("values : values SPACE VALUE")
def values_values(p):
return p[0] + [p[2].getstr()]
@pg.production("none :")
def none(p):
return None
parser = pg.build()
assert parser.parse(FakeLexer([
Token("VALUE", "abc"),
Token("SPACE", " "),
Token("VALUE", "def"),
Token("SPACE", " "),
Token("VALUE", "ghi"),
])) == ["abc", "def", "ghi"]
assert parser.parse(FakeLexer([])) == []
def test_arithmetic(self):
lg = LexerGenerator()
lg.add("NUMBER", r"\d+")
lg.add("PLUS", r"\+")
lg.add("TIMES", r"\*")
pg = ParserGenerator(["NUMBER", "PLUS", "TIMES"],
precedence=[
("left", ["PLUS"]),
("left", ["TIMES"]),
])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
@pg.production("expr : expr TIMES expr")
def expr_binop(p):
return BoxInt({
"+": operator.add,
"*": operator.mul
}[p[1].getstr()](p[0].getint(), p[2].getint()))
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
lexer = lg.build()
parser = pg.build()
assert parser.parse(lexer.lex("3*4+5"))
def __init__(self):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
[
'SCANF', 'OPEN_PAREN', 'CLOSE_PAREN', 'POINT', 'PLUS', 'COMMA',
'SEMI_COLON', 'CONNECTION_CLASS', 'EXEC_FUNC', 'VAR', 'STRING'
])
def test_precedence(self):
pg = ParserGenerator(["NUMBER", "PLUS", "TIMES"], precedence=[
("left", ["PLUS"]),
("left", ["TIMES"]),
])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
@pg.production("expr : expr TIMES expr")
def expr_binop(p):
return BoxInt({
"+": operator.add,
"*": operator.mul
}[p[1].getstr()](p[0].getint(), p[2].getint()))
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
parser = pg.build()
assert parser.parse(FakeLexer([
Token("NUMBER", "3"),
Token("TIMES", "*"),
Token("NUMBER", "4"),
Token("PLUS", "+"),
Token("NUMBER", "5")
])) == BoxInt(17)
def test_state(self):
pg = ParserGenerator(["NUMBER", "PLUS"], precedence=[
("left", ["PLUS"]),
])
@pg.production("main : expression")
def main(state, p):
state.count += 1
return p[0]
@pg.production("expression : expression PLUS expression")
def expression_plus(state, p):
state.count += 1
return BoxInt(p[0].getint() + p[2].getint())
@pg.production("expression : NUMBER")
def expression_number(state, p):
state.count += 1
return BoxInt(int(p[0].getstr()))
parser = pg.build()
state = ParserState()
assert parser.parse(iter([
Token("NUMBER", "10"),
Token("PLUS", "+"),
Token("NUMBER", "12"),
Token("PLUS", "+"),
Token("NUMBER", "-2"),
]), state=state) == BoxInt(20)
assert state.count == 6
def test_arithmetic(self):
pg = ParserGenerator(["NUMBER", "PLUS"])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
def expr_op(p):
return BoxInt(p[0].getint() + p[2].getint())
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
with self.assert_warns(
ParserGeneratorWarning, "1 shift/reduce conflict"
):
parser = pg.build()
assert parser.parse(iter([
Token("NUMBER", "1"),
Token("PLUS", "+"),
Token("NUMBER", "4")
])) == BoxInt(5)
def test_per_rule_precedence(self):
pg = ParserGenerator(["NUMBER", "MINUS"], precedence=[
("right", ["UMINUS"]),
])
@pg.production("main : expr")
def main_expr(p):
return p[0]
@pg.production("expr : expr MINUS expr")
def expr_minus(p):
return BoxInt(p[0].getint() - p[2].getint())
@pg.production("expr : MINUS expr", precedence="UMINUS")
def expr_uminus(p):
return BoxInt(-p[1].getint())
@pg.production("expr : NUMBER")
def expr_number(p):
return BoxInt(int(p[0].getstr()))
with self.assert_warns(
ParserGeneratorWarning, "1 shift/reduce conflict"
):
parser = pg.build()
assert parser.parse(iter([
Token("MINUS", "-"),
Token("NUMBER", "4"),
Token("MINUS", "-"),
Token("NUMBER", "5"),
])) == BoxInt(-9)
def test_null_production(self):
pg = ParserGenerator(["VALUE", "SPACE"])
@pg.production("main : values")
def main(p):
return p[0]
@pg.production("values : none")
def values_empty(p):
return []
@pg.production("values : VALUE")
def values_value(p):
return [p[0].getstr()]
@pg.production("values : values SPACE VALUE")
def values_values(p):
return p[0] + [p[2].getstr()]
@pg.production("none :")
def none(p):
return None
parser = pg.build()
assert parser.parse(iter([
Token("VALUE", "abc"),
Token("SPACE", " "),
Token("VALUE", "def"),
Token("SPACE", " "),
Token("VALUE", "ghi"),
])) == ["abc", "def", "ghi"]
assert parser.parse(iter([])) == []
def test_duplicate_precedence(self):
pg = ParserGenerator([], precedence=[
("left", ["term", "term"])
])
with py.test.raises(ParserGeneratorError):
pg.build()
def test_invalid_associativity(self):
pg = ParserGenerator([], precedence=[
("to-the-left", ["term"]),
])
with py.test.raises(ParserGeneratorError):
pg.build()
def __init__(self, dic_variables, dic_etiquetas):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
['LODD','STOD','ADDD','SUBD','JPOS','JZER','JUMP','LOCO','LODL','STOL','ADDL','SUBL','JNEG','JNZE','CALL','PUSHI','POPI','PUSH','POP','RETN','SWAP','INSP','DESP','INPAC', 'OUTAC','HALT','DIRECCION', 'NUMERO', 'VARIABLE', 'ETIQUETA']
)
self.dic_variables=dic_variables
self.dic_etiquetas=dic_etiquetas
def __init__(self):
self.pg = ParserGenerator([
'NUMBER', 'OPEN_PARENS', 'CLOSE_PARENS', 'SEMI_COLON', 'SUM', 'SUB'
],
precedence=[
('left', ['SUM', 'SUB']),
])
def __init__(self, syntax=False):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
get_all_tokens_name(),
# A list of precedence rules with ascending precedence, to
# disambiguate ambiguous production rules.
precedence=((AppConstant.LEFT, [TokenEnum.FUNCTION.name]),
(AppConstant.LEFT, [TokenEnum.LET.name]),
(AppConstant.LEFT,
[TokenEnum.ASSIGN.name]), (AppConstant.LEFT, [
TokenEnum.IF.name, TokenEnum.ELSE.name,
TokenEnum.SEMI_COLON.name
]), (AppConstant.LEFT,
[TokenEnum.AND.name, TokenEnum.OR.name
]), (AppConstant.LEFT, [TokenEnum.NOT.name]),
(AppConstant.LEFT, [
TokenEnum.EQ.name, TokenEnum.NEQ.name,
TokenEnum.GTEQ.name, TokenEnum.GT.name,
TokenEnum.LT.name, TokenEnum.LTEQ.name
]), (AppConstant.LEFT,
[TokenEnum.SUM.name, TokenEnum.SUB.name]),
(AppConstant.LEFT,
[TokenEnum.MUL.name,
TokenEnum.DIV.name]), (AppConstant.LEFT, [
TokenEnum.STRING.name, TokenEnum.INTEGER.name,
TokenEnum.FLOAT.name, TokenEnum.BOOLEAN.name,
TokenEnum.PI.name, TokenEnum.E.name
])))
self.syntax = syntax
self.parse()
pass # End Parser's constructor !
def __init__(self):
self.pg = ParserGenerator([
'NUMBER',
'STRING',
'IDENT',
'PRINT',
'READ',
'INT',
'IF',
'THEN',
'ELSE',
'END_IF',
'FOR',
'UNTIL',
'STEP',
'END_FOR',
'OPEN_PAREN',
'CLOSE_PAREN',
'END_STATEMENT',
'DECLARE_TYPE',
'ADD',
'SUB',
'MULT',
'DIV',
'LESS',
'MORE',
'LESS_EQ',
'MORE_EQ',
],
precedence=[('left', ['ADD', 'SUB']),
('left', ['MUL', 'DIV'])])
self.vars = {}
def __init__(self, module, builder, printf):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
['RESERVED', 'END', 'DIGIT', 'LETTER', 'SPECIAL'])
self.module = module
self.builder = builder
self.printf = printf
def test_duplicate_precedence(self):
pg = ParserGenerator([], precedence=[
("left", ["term", "term"])
])
with py.test.raises(ParserGeneratorError):
pg.build()
def __init__(self):
self.indent_count = 0
self.definition_list = []
self.pg = ParserGenerator(
# A list of all token names, accepted by the parser.
[
'INTEGER', 'IDENTIFIER', 'IF', 'ELSE', 'COLON', 'NOT', 'WHILE',
'END', 'FUNCTION', 'OPENPAREN', 'CLOSEPAREN', 'NOT', 'IMPORT',
'BEGIN', 'MOVE', 'LEFTTURN', 'PUTBEEPER', 'PICKBEEPER',
'FACENORTH', 'FACESOUTH', 'FACEWEST', 'LEFTCLEAR',
'RIGHTCLEAR', 'FRONTCLEAR', 'PRESENT', 'INBAG', 'FOR',
'FACEEAST', 'INDENT', 'DEDENT'
],
# A list of precedence rules with ascending precedence, to
# disambiguate ambiguous production rules.
precedence=[('left', [
'FUNCTION',
]), ('left', ['[', ']', ',']),
('left', [
'IF',
'COLON',
'ELSE',
' ',
'WHILE',
])])
def test_basic_parser(self):
pg = ParserGenerator(["NUMBER", "PLUS"])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
def expr_op(p):
return BoxInt(p[0].getint() + p[2].getint())
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
with self.assert_warns(ParserGeneratorWarning,
"1 shift/reduce conflict"):
parser = pg.build()
def f(n):
return parser.parse(
FakeLexer([
Token("NUMBER", str(n)),
Token("PLUS", "+"),
Token("NUMBER", str(n))
])).getint()
assert self.run(f, [12]) == 24
def __init__(self,
lexer_or_tokens: Union[Lexer, Iterable[str]],
precedence: Optional[ParserPrecedence] = None):
self._pg = ParserGenerator(
lexer_or_tokens.possible_tokens if isinstance(
lexer_or_tokens, Lexer) else lexer_or_tokens, precedence or [])
self._pg.error(self._handle_error)
def __init__(self, module, builder, printf):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
[
'NUMERO',
'ESCREVA',
'APAR',
'FPAR',
'PONTO_VIRGULA',
'SOMA',
'SUB',
'MUL',
'DIV',
],
precedence=[
('left', [
'SOMA',
'SUB',
]),
('left', [
'MUL',
'DIV',
]),
],
)
self.module = module
self.builder = builder
self.printf = printf
def __init__(self):
self.pg = ParserGenerator([
'ADD', 'COLON', 'PREDICTION', 'CLUSTERING', 'FLOAT', 'STATS',
'PLOT', 'C_PLOT', 'SENTENCE', 'PRINT', 'FORECAST', 'VAR', 'EQUAL',
'INT', 'OPEN_BRA', 'CLOSE_BRA', 'OPEN_PAREN', 'CLOSE_PAREN',
'SEMI_COLON', 'COMA', 'METHOD', 'BOOLEAN', 'TYPE_A'
])
def test_invalid_associativity(self):
pg = ParserGenerator([], precedence=[
("to-the-left", ["term"]),
])
with py.test.raises(ParserGeneratorError):
pg.build()
def __init__(self):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
[
'NUMBER', 'PRINT', 'OPEN_PAREN', 'CLOSE_PAREN', 'SEMI_COLON',
'SUM', 'SUB', 'OPEN_QUOTE', 'CLOSE_QUOTE'
])
def __init__(self):
self.token_list = ["NUMBER", "PLUS", "EXIT"]
pg = ParserGenerator(self.token_list, cache_id='wcc')
@pg.production("main : statement")
def main(p):
return p[0]
@pg.production("statement : addition")
@pg.production("statement : exit")
def main(p):
return p[0]
@pg.production("exit : EXIT")
def exit_prod(p):
return ExitBox()
@pg.production("number : NUMBER")
def number(p):
return IntBox(p[0])
@pg.production("addition : PLUS number number")
def addition0(p):
return BinaryAddBox(p[1], p[2])
@pg.production("addition : number PLUS number")
def addition1(p):
return BinaryAddBox(p[0], p[2])
@pg.production("addition : number number PLUS")
def addition2(p):
return BinaryAddBox(p[0], p[1])
self.parser = pg.build()
def test_parser_state(self):
pg = ParserGenerator(["NUMBER", "PLUS"],
precedence=[
("left", ["PLUS"]),
])
@pg.production("main : expression")
def main(state, p):
state.count += 1
return p[0]
@pg.production("expression : expression PLUS expression")
def expression_plus(state, p):
state.count += 1
return BoxInt(p[0].getint() + p[2].getint())
@pg.production("expression : NUMBER")
def expression_number(state, p):
state.count += 1
return BoxInt(int(p[0].getstr()))
parser = pg.build()
def f():
state = ParserState()
return parser.parse(FakeLexer([
Token("NUMBER", "10"),
Token("PLUS", "+"),
Token("NUMBER", "12"),
Token("PLUS", "+"),
Token("NUMBER", "-2"),
]),
state=state).getint() + state.count
assert self.run(f, []) == 26
def test_arithmetic(self):
pg = ParserGenerator(["NUMBER", "PLUS"])
@pg.production("main : expr")
def main(p):
return p[0]
@pg.production("expr : expr PLUS expr")
def expr_op(p):
return BoxInt(p[0].getint() + p[2].getint())
@pg.production("expr : NUMBER")
def expr_num(p):
return BoxInt(int(p[0].getstr()))
with self.assert_warns(
ParserGeneratorWarning, "1 shift/reduce conflict"
):
parser = pg.build()
assert parser.parse(iter([
Token("NUMBER", "1"),
Token("PLUS", "+"),
Token("NUMBER", "4")
])) == BoxInt(5)
def test_per_rule_precedence(self):
pg = ParserGenerator(["NUMBER", "MINUS"], precedence=[
("right", ["UMINUS"]),
])
@pg.production("main : expr")
def main_expr(p):
return p[0]
@pg.production("expr : expr MINUS expr")
def expr_minus(p):
return BoxInt(p[0].getint() - p[2].getint())
@pg.production("expr : MINUS expr", precedence="UMINUS")
def expr_uminus(p):
return BoxInt(-p[1].getint())
@pg.production("expr : NUMBER")
def expr_number(p):
return BoxInt(int(p[0].getstr()))
with self.assert_warns(ParserGeneratorWarning, "1 shift/reduce conflict"):
parser = pg.build()
assert parser.parse(FakeLexer([
Token("MINUS", "-"),
Token("NUMBER", "4"),
Token("MINUS", "-"),
Token("NUMBER", "5"),
])) == BoxInt(-9)
def __init__(self, module, builder, printf):
self.pg = ParserGenerator([
'NUMBER', 'PRINT', 'OPEN_PAREN', 'CLOSE_PAREN', 'SEMI_COLON',
'ADD', 'SUB'
])
self.module = module
self.builder = builder
self.printf = printf
def __init__(self):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
[
'NOMBRE', 'MON', 'PARENTESE1', 'PARENTESE2', 'POINT_VERG',
'PLUS', 'MOINS', 'FOIS', 'DIVI', 'TERM', 'EGAL', 'QUOTE',
'VERGULE', 'DOLLAR'
])
def __init__(self):
self.pg = ParserGenerator(
[
'SELECT', 'FROM', 'WHERE', 'SEP', 'BOXPLOT', 'STR', 'GRT',
'LSS', 'EQ'
]
#Adicionar o NUM no final, agora é tudo string
)
def __init__(self):
self.pg = ParserGenerator(
# A list of all token names accepted by the parser.
[
'NUMBER', 'PRINTTOLCD', 'OPEN_PAREN', 'CLOSE_PAREN',
'SEMI_COLON', 'SUM', 'SUB', 'STRING', 'SLEEP', 'OUTPUT',
'PINON', 'PINOFF', 'EQUAL', 'VAR', 'VARNAME', 'GETVAR'
])
def __init__(self):
self.parseGen = ParserGenerator(
# A list of all token names accepted by the parser.
[
'NUMBER', 'PRINT', 'LEFT_PAREN', 'RIGHT_PAREN', 'EOL', 'SUM',
'SUB', 'MUL', 'DIV'
],
precedence=[('left', ['SUM', 'SUB']), ('left', ['MUL', 'DIV'])])
def test_nonexistant_precedence(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : VALUE", precedence="abc")
def main(p):
pass
with py.test.raises(ParserGeneratorError):
pg.build()
def test_production_terminal_overlap(self):
pg = ParserGenerator(["VALUE"])
@pg.production("VALUE :")
def x(p):
pass
with py.test.raises(ParserGeneratorError):
pg.build()
def test_unused_tokens(self):
pg = ParserGenerator(["VALUE", "OTHER"])
@pg.production("main : VALUE")
def main(p):
return p[0]
with self.assert_warns(ParserGeneratorWarning, "Token 'OTHER' is unused"):
pg.build()
def test_simple(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : VALUE")
def main(p):
return p[0]
parser = pg.build()
assert parser.parse(iter([Token("VALUE", "abc")])) == Token("VALUE", "abc")
def build():
pg = ParserGenerator(TOKENS, cache_id="sql_parser")
for prod, cls in gen_productions(select_bnf, node_classes):
pg.production(prod)(cls.production)
# print prod
@pg.error
def error_handler(token):
raise ValueError("Ran into a %s(%s) where it wasn't expected"
% (token.gettokentype(), token.getstr()))
return pg.build()
def test_error_symbol(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : VALUE")
def main(p):
pass
@pg.production("main : error")
def main_error(p):
pass
pg.build()
def test_unused_production(self):
pg = ParserGenerator(["VALUE", "OTHER"])
@pg.production("main : VALUE")
def main(p):
return p[0]
@pg.production("unused : OTHER")
def unused(p):
pass
with self.assert_warns(ParserGeneratorWarning, "Production 'unused' is not reachable"):
pg.build()
def test_parse_error(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : VALUE")
def main(p):
return p[0]
parser = pg.build()
with py.test.raises(ParsingError) as exc_info:
parser.parse(FakeLexer([Token("VALUE", "hello"), Token("VALUE", "world", SourcePosition(5, 10, 2))]))
assert exc_info.value.getsourcepos().lineno == 10
def test_simple_caching(self):
pg = ParserGenerator(["VALUE"], cache_id="simple")
@pg.production("main : VALUE")
def main(p):
return p[0]
pg.build()
parser = pg.build()
assert parser.parse(iter([
Token("VALUE", "3")
])) == Token("VALUE", "3")
def test_simple(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : VALUE")
def main(p):
return p[0]
parser = pg.build()
assert parser.lr_table.lr_action == {
0: {"VALUE": 2},
1: {"$end": 0},
2: {"$end": -1},
}
def test_simple_caching(self):
# Generate a random cache_id so that every test run does both the cache
# write and read paths.
pg = ParserGenerator(["VALUE"], cache_id=str(uuid.uuid4()))
@pg.production("main : VALUE")
def main(p):
return p[0]
pg.build()
parser = pg.build()
assert parser.parse(iter([
Token("VALUE", "3")
])) == Token("VALUE", "3")
def parser_from_lexer(lexer, mapping):
pg = ParserGenerator(
[rule.name for rule in lexer.rules],
cache_id="cache",
# NOTE: This is pretty arbitrary at the moment
precedence=[
('right', ['NOT']),
('left', ['PARENT', 'CHILD', 'OR']),
('left', ['AND', 'TYPE']),
('left', ['L_PAREN', 'R_PAREN', 'EQUAL']),
])
@pg.production("expr : L_PAREN expr R_PAREN")
def expr(p):
return p[1]
@pg.production("expr : expr AND expr")
@pg.production("expr : expr OR expr")
@pg.production("expr : expr CHILD expr")
@pg.production("expr : expr PARENT expr")
def binary_operation(p):
return [p[1], p[0], p[2]]
@pg.production("expr : expr EQUAL DATA")
@pg.production("expr : expr TYPE DATA")
def equal(p):
p[2].value = p[2].value.strip("/")
return p
@pg.production("expr : expr NOT CHILD expr")
@pg.production("expr : expr NOT PARENT expr")
def not_expr(p):
# This is a hack
op = p[2]
op.name = "NOT_" + op.name
op.value = "!" + op.value
return [op, p[0], p[3]]
for kind in mapping.keys():
@pg.production("expr : " + kind)
def kind(p):
return mapping[p[0].name]
parser = pg.build()
return parser
def test_default_reductions(self):
pg = ParserGenerator(
["INTEGER_START", "INTEGER_VALUE", "COMPARE"],
precedence=[
("nonassoc", ["COMPARE"])
]
)
record = []
@pg.production("main : expr")
def main(p):
record.append("main")
return p[0]
@pg.production("expr : expr COMPARE expr")
def expr_compare(p):
record.append("expr:compare")
return BoxInt(p[0].getint() - p[2].getint())
@pg.production("expr : INTEGER_START INTEGER_VALUE")
def expr_int(p):
record.append("expr:int")
return BoxInt(int(p[1].getstr()))
parser = pg.build()
assert parser.parse(RecordingLexer(record, [
Token("INTEGER_START", ""),
Token("INTEGER_VALUE", "10"),
Token("COMPARE", "-"),
Token("INTEGER_START", ""),
Token("INTEGER_VALUE", "5")
])) == BoxInt(5)
assert record == [
"token:INTEGER_START",
"token:INTEGER_VALUE",
"expr:int",
"token:COMPARE",
"token:INTEGER_START",
"token:INTEGER_VALUE",
"expr:int",
"expr:compare",
"token:None",
"main",
]
def test_reduce_reduce(self):
pg = ParserGenerator(["NAME", "EQUALS", "NUMBER"])
@pg.production("main : assign")
def main(p):
pass
@pg.production("assign : NAME EQUALS expression")
@pg.production("assign : NAME EQUALS NUMBER")
def assign(p):
pass
@pg.production("expression : NUMBER")
def expression(p):
pass
with self.assert_warns(ParserGeneratorWarning, "1 reduce/reduce conflict"):
pg.build()
def test_parse_error_handler(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : VALUE")
def main(p):
return p[0]
@pg.error
def error_handler(token):
raise ValueError(token)
parser = pg.build()
token = Token("VALUE", "world")
with py.test.raises(ValueError) as exc_info:
parser.parse(FakeLexer([Token("VALUE", "hello"), token]))
assert exc_info.value.args[0] is token
def test_error_handler_state(self):
pg = ParserGenerator([])
@pg.production("main :")
def main(state, p):
pass
@pg.error
def error(state, token):
raise ValueError(state, token)
parser = pg.build()
state = ParserState()
token = Token("VALUE", "")
with py.test.raises(ValueError) as exc_info:
parser.parse(iter([token]), state=state)
assert exc_info.value.args[0] is state
assert exc_info.value.args[1] is token
def test_shift_reduce(self):
pg = ParserGenerator([
"NAME", "NUMBER", "EQUALS", "PLUS", "MINUS", "TIMES", "DIVIDE",
"LPAREN", "RPAREN"
])
@pg.production("statement : NAME EQUALS expression")
def statement_assign(p):
pass
@pg.production("statement : expression")
def statement_expression(p):
pass
@pg.production("expression : expression PLUS expression")
@pg.production("expression : expression MINUS expression")
@pg.production("expression : expression TIMES expression")
@pg.production("expression : expression DIVIDE expression")
def expression_binop(p):
pass
@pg.production("expression : MINUS expression")
def expression_uminus(p):
pass
@pg.production("expression : LPAREN expression RPAREN")
def expression_group(p):
pass
@pg.production("expression : NUMBER")
def expression_number(p):
pass
@pg.production("expression : NAME")
def expression_name(p):
pass
with self.assert_warns(
ParserGeneratorWarning, "20 shift/reduce conflicts"
):
pg.build()
def test_empty_production(self):
pg = ParserGenerator(["VALUE"])
@pg.production("main : values")
def main(p):
return p[0]
@pg.production("values : VALUE values")
def values_value(p):
return [p[0]] + p[1]
@pg.production("values :")
def values_empty(p):
return []
parser = pg.build()
assert parser.lr_table.lr_action == {
0: {"$end": -3, "VALUE": 3},
1: {"$end": 0},
2: {"$end": -1},
3: {"$end": -3, "VALUE": 3},
4: {"$end": -2},
}
def test_production_syntax_error(self):
pg = ParserGenerator([])
with py.test.raises(ParserGeneratorError):
pg.production("main VALUE")
lg.add("char_return", r"\\return")
lg.add("char_space", r"\\space")
lg.add("char", r"\\.")
lg.add("ns_symbol", NS_SYMBOL)
lg.add("symbol", SYMBOL_RE)
lg.add("string", r'"(\\\^.|\\.|[^\"])*"')
lg.add("ns_tag", "#" + NS_SYMBOL)
lg.add("tag", "#" + SYMBOL_RE)
lg.ignore(r"[\s,\n]+")
lg.ignore(r";.*\n")
lexer = lg.build()
pg = ParserGenerator(["boolean", "nil", "float", "number", "olist", "clist",
"omap", "cmap", "ovec", "cvec", "oset", "colon", "char_nl", "char_tab",
"char_return", "char_space", "char", "symbol", "ns_symbol", "string",
"tag", "ns_tag"])
class Char(TaggedValue):
def __init__(self, rep):
TaggedValue.__init__(self, 'char', rep)
NL = Char('\n')
TAB = Char('\t')
RETURN = Char('\r')
SPACE = Char(' ')
class State(object):
def __init__(self, tagged, accept_unknown_tags):
self.tagged = tagged if tagged is not None else {}
self.accept_unknown_tags = accept_unknown_tags
lex = LexerGenerator()
lex.ignore(ur'(?:[,;\s]+|\band\b|\bor\b)+')
lex.add(u'URL', UrlPattern)
lex.add(u'BTHASH', ur'#betterthan')
lex.add(u'IBTHASH', ur'#isbetterthan')
lex.add(u'HASHTAG', ur'#[a-zA-Z0-9_]+')
lex.add(u'MENTION', ur'@[a-zA-Z0-9_]+')
lex.add(u'FOR', ur'(for|FOR|For)')
lex.add(u'WORD', ur'[\w]+')
pg = ParserGenerator([u'URL',
u'BTHASH',
u'IBTHASH',
u'HASHTAG',
u'MENTION',
u'FOR',
u'WORD'
],
cache_id=u'graphextractor.tweetparser')
@pg.production("betterthan : words URL bthash URL topics words")
def betterthan(p):
ast = dict()
ast.update(p[4])
ast.update(better_url=p[1].value)
ast.update(worse_url=p[3].value)
return ast
@pg.production("bthash : BTHASH")
@pg.production("bthash : IBTHASH")