def _parse(self, input):
cStream = antlr3.StringStream(input)
lexer = self.getLexer(cStream)
tStream = antlr3.TokenRewriteStream(lexer)
tStream.fillBuffer()
return tStream
def testRewrite(self):
self.compileGrammar()
input = textwrap.dedent(
'''\
method foo() {
i = 3;
k = i;
i = k*4;
}
method bar() {
j = i*2;
}
''')
cStream = antlr3.StringStream(input)
lexer = self.getLexer(cStream)
tStream = antlr3.TokenRewriteStream(lexer)
parser = self.getParser(tStream)
parser.program()
expectedOutput = textwrap.dedent('''\
public class Wrapper {
public void foo() {
int k;
int i;
i = 3;
k = i;
i = k*4;
}
public void bar() {
int j;
j = i*2;
}
}
''')
self.assertEqual(str(tStream), expectedOutput)
def testTreeRewrite(self):
grammar = textwrap.dedent(r'''grammar T6;
options {
language=Python;
output=AST;
}
tokens {
BLOCK;
ASSIGN;
}
prog: stat+;
stat
: IF '(' e=expr ')' s=stat
-> ^(IF $e $s)
| RETURN expr ';'
-> ^(RETURN expr)
| '{' stat* '}'
-> ^(BLOCK stat*)
| ID '=' expr ';'
-> ^(ASSIGN ID expr)
;
expr
: ID
| INT
;
IF: 'if';
RETURN: 'return';
ID: 'a'..'z'+;
INT: '0'..'9'+;
WS: (' '|'\n')+ {$channel=HIDDEN;} ;
COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ;
''')
treeGrammar = textwrap.dedent(r'''tree grammar T6Walker;
options {
language=Python;
tokenVocab=T6;
ASTLabelType=CommonTree;
output=template;
rewrite=true;
}
prog: stat+;
stat
: ^(IF expr stat)
| ^(RETURN return_expr)
| ^(BLOCK stat*)
| ^(ASSIGN ID expr)
;
return_expr
: expr
-> template(t={$text}) <<boom(<t>)>>
;
expr
: ID
| INT
;
''')
input = textwrap.dedent('''\
if ( foo ) {
b = /* bla */ 2;
return 1 /* foo */;
}
/* gnurz */
return 12;
''')
lexerCls, parserCls = self.compileInlineGrammar(grammar)
walkerCls = self.compileInlineGrammar(treeGrammar)
cStream = antlr3.StringStream(input)
lexer = lexerCls(cStream)
tStream = antlr3.TokenRewriteStream(lexer)
parser = parserCls(tStream)
tree = parser.prog().tree
nodes = antlr3.tree.CommonTreeNodeStream(tree)
nodes.setTokenStream(tStream)
walker = walkerCls(nodes)
walker.prog()
found = tStream.toString()
expected = textwrap.dedent('''\
if ( foo ) {
b = /* bla */ 2;
return boom(1) /* foo */;
}
/* gnurz */
return boom(12);
''')
self.failUnlessEqual(expected, found)
def testRewrite(self):
grammar = textwrap.dedent(r'''grammar T5;
options {
language=Python;
output=template;
rewrite=true;
}
prog: stat+;
stat
: 'if' '(' expr ')' stat
| 'return' return_expr ';'
| '{' stat* '}'
| ID '=' expr ';'
;
return_expr
: expr
-> template(t={$text}) <<boom(<t>)>>
;
expr
: ID
| INT
;
ID: 'a'..'z'+;
INT: '0'..'9'+;
WS: (' '|'\n')+ {$channel=HIDDEN;} ;
COMMENT: '/*' (options {greedy=false;} : .)* '*/' {$channel = HIDDEN;} ;
''')
input = textwrap.dedent('''\
if ( foo ) {
b = /* bla */ 2;
return 1 /* foo */;
}
/* gnurz */
return 12;
''')
lexerCls, parserCls = self.compileInlineGrammar(grammar)
cStream = antlr3.StringStream(input)
lexer = lexerCls(cStream)
tStream = antlr3.TokenRewriteStream(lexer)
parser = parserCls(tStream)
result = parser.prog()
found = tStream.toString()
expected = textwrap.dedent('''\
if ( foo ) {
b = /* bla */ 2;
return boom(1) /* foo */;
}
/* gnurz */
return boom(12);
''')
self.failUnlessEqual(expected, found)
import sys import antlr3 from TLexer import TLexer from TParser import TParser cStream = antlr3.FileStream(sys.argv[1]) lexer = TLexer(cStream) tStream = antlr3.TokenRewriteStream(lexer) parser = TParser(tStream) parser.program() print tStream