def execParser(self,
grammar,
grammarEntry,
input,
listener,
parser_args={}):
if listener is None:
port = 49100
debugger = Debugger(port)
debugger.start()
# TODO(pink): install alarm, so it doesn't hang forever in case of a bug
else:
port = None
try:
lexerCls, parserCls = self.compileInlineGrammar(grammar,
options='-debug')
cStream = antlr3.StringStream(input)
lexer = lexerCls(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = parserCls(tStream, dbg=listener, port=port, **parser_args)
getattr(parser, grammarEntry)()
finally:
if listener is None:
debugger.join()
return debugger
def testLBBeyondBegin(self):
"""CommonTokenStream.LB(-1): beyond begin"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
)
self.source.tokens.append(
antlr3.CommonToken(type=12, channel=antlr3.HIDDEN_CHANNEL)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
stream = antlr3.CommonTokenStream(self.source)
self.failUnless(stream.LB(1) is None)
stream.consume()
stream.consume()
self.failUnless(stream.LB(3) is None)
def convert_charstream(char,verbose=0):
lexer = JavaScriptLexer(char)
tokens = antlr3.CommonTokenStream(lexer)
pp = JavaScriptParser(tokens)
prgm = pp.program()
tree = prgm.tree
return convert_tree(tree,verbose)
def parse(string, noprint=False):
cStream = antlr3.StringStream(string)
lexer = foo_langLexer(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = foo_langParser(tStream)
try:
return parser.start()
except RecognitionException as e:
if not noprint:
print "Exception:", e, ":"
print " index :", e.index
print " token :", e.token
print " c :", e.c
print " line :", e.line
lines = string.split("\n")
print " ", lines[e.line-2]
print " -->", lines[e.line-1]
if e.line < len(lines):
print " ", lines[e.line]
print " pos :", e.charPositionInLine
print " info :", e.approximateLineInfo
raise RuntimeError("Failed to parse")
else:
raise
def sql2sparkScaffold(self, relative_path):
'''
Parses the given SQL file and compares the parse tree with the expected parse tree
'''
errorMsg = """Expected output and produced output do not match for %s:
Expected output: %s
---------------------------------------------------------------
Produced output: %s
---------------------------------------------------------------
Diff: %s
==============================================================="""
path = os.path.join(base_tests_path, relative_path)
with open(path) as sqlFile:
# TODO Meisam: Make the grammar case insensitive?
query = sqlFile.read().upper()
stringStream = antlr3.StringStream(query)
lexer = YSmartLexer(stringStream)
tokenStream = antlr3.CommonTokenStream(lexer)
parser = YSmartParser(tokenStream)
parse_tree = parser.start_rule()
graphviz = visit_tree(parse_tree.tree)
with open(path + ".dot", 'w') as dot_file:
dot_file.write(graphviz)
dot_file.flush()
dot_file.close()
def testValid1(self):
cStream = antlr3.StringStream('int a;')
lexer = self.getLexer(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = self.getParser(tStream)
events = parser.translation_unit()
def testValid(self):
cStream = antlr3.StringStream("int foo;")
lexer = self.getLexer(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = self.getParser(tStream)
name = parser.declaration()
assert name == 'foo', name
def testMarkRewind(self):
"""CommonTokenStream.mark()/rewind()"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
self.source.tokens.append(
antlr3.CommonToken(type=antlr3.EOF)
)
stream = antlr3.CommonTokenStream(self.source)
stream.fillBuffer()
stream.consume()
marker = stream.mark()
stream.consume()
stream.rewind(marker)
self.failUnlessEqual(stream.LA(1), 13)
def parse_func(pfunc):
char_stream = antlr3.ANTLRStringStream(pfunc)
lexer = PyFuncLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
tokens.fillBuffer()
parser = PyFuncParser(tokens)
return parser.pyfunc()
def do_test(self, dgdl):
"""Test the given dgdl specification
:param dgdl the DGDL specification
:type dgdl str
"""
try:
'''To catch parse errors, redirect the output of sys.stderr
to a variable'''
old_stderr = sys.stderr
sys.stderr = mystdout = StringIO()
'''Attempt to parse the DGDL spec with antlr'''
char_stream = antlr3.ANTLRStringStream(dgdl)
lexer = dgdlLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = dgdlParser(tokens)
tree = parser.system().tree
''' Reassign sys.stderr to its previous location '''
sys.stderr = old_stderr
'''Return value will be any parse errors, or an empty string if
no errors'''
return mystdout.getvalue()
except Exception as e:
''' This is an error with even trying to parse, not a parse error'''
return None
def main(argv=sys.argv):
inpath, outpath = argv[1:]
with open(inpath, 'rb') as inf:
char_stream = antlr3.ANTLRInputStream(inf)
lexer = ZasLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = ZasParser(tokens)
r = parser.program()
t = r.tree
#print t.toStringTree()
nodes = antlr3.tree.CommonTreeNodeStream(t)
nodes.setTokenStream(tokens)
assembler = ZasAssembler()
walker = ZasWalker(nodes, assembler)
walker.program()
assembler.finalize()
zcode = []
for secname in ('data', 'rodata', 'text'):
zcode.append(assembler.sections[secname].getvalue())
zcode = ''.join(zcode)[0x40:]
header = ZHeader()
header.version = 5
header.initpc = assembler.start
header.globals = assembler.globals
header.statmem = assembler.sections['rodata'].base
header.himem = assembler.sections['text'].base
header.filesz = len(zcode) + 0x40
with open(outpath, 'wb') as outf:
outf.write(str(header))
outf.write(zcode)
return 0
def CreateParser(query):
"""Creates a Query Parser."""
input_string = antlr3.ANTLRStringStream(query)
lexer = QueryLexerWithErrors(input_string)
tokens = antlr3.CommonTokenStream(lexer)
parser = QueryParserWithErrors(tokens)
return parser
def CreateParser(expression):
"""Creates a Expression Parser."""
input_string = antlr3.ANTLRStringStream(expression)
lexer = ExpressionLexerWithErrors(input_string)
tokens = antlr3.CommonTokenStream(lexer)
parser = ExpressionParserWithErrors(tokens)
return parser
def parseFSM(path):
char_stream = antlr3.ANTLRInputStream(open(path))
lexer = FsmlLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = FsmlParser(tokens)
parser.fsm()
return parser.fsmObject
def parse(self,text):
if(type(text) is not type("ü".decode('UTF-8'))):
logger.info("converting input string to utf-8")
text = text.decode('UTF-8')
logger.debug(text)
char_stream = antlr3.ANTLRStringStream(text)
try:
lexer = cp_lexer(char_stream)
tokenstream = antlr3.CommonTokenStream(lexer)
parser = cp_parser(tokenstream)
try:
r=parser.doc()
root = r.tree
print(root.toStringTree())
logger.debug(root.toStringTree())
try:
nodes = antlr3.tree.CommonTreeNodeStream(root)
nodes.setTokenStream(tokenstream)
tp = cp_treeparser(nodes)
tp.doc()
logger.debug(tp.refs)
json_string = json.dumps(tp.refs)
return json.loads(json_string)
except Exception as e:
logger.error("%s"%e)
logger.error("there was a problem w/ the TreeParser: exiting")
except Exception as e:
logger.error("%s"%e)
logger.error("there was a problem with the parser: exiting")
except Exception as e:
logger.error("%s"%e)
logger.error("there was a problem with the lexer: exiting")
def testConsume(self):
"""CommonTokenStream.consume()"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
self.source.tokens.append(
antlr3.CommonToken(type=antlr3.EOF)
)
stream = antlr3.CommonTokenStream(self.source)
self.failUnlessEqual(stream.LA(1), 12)
stream.consume()
self.failUnlessEqual(stream.LA(1), 13)
stream.consume()
self.failUnlessEqual(stream.LA(1), antlr3.EOF)
stream.consume()
self.failUnlessEqual(stream.LA(1), antlr3.EOF)
def testValid4(self):
cStream = antlr3.StringStream('bar name1 name2')
lexer = self.getLexer(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = self.getParser(tStream)
parser.cond = False
events = parser.r()
def __init__(self, file="", filename="", graphexport=False, dbg_enabled: bool = False):
Debug.__init__(self, dbg_enabled)
# PROTO DATA OBJECTS ####
self.constNode: Dict[str, CommonTree] = {}
self.networkNode: List[CommonTree] = []
# Architecture nodes
self.cache_node: PCCObject = None
self.dir_node: PCCObject = None
self.mem_node: PCCObject = None
self.msgNode: List[PCCObject] = []
self.msgTypes: List[str] = []
self.dataMsgTypes: List[str] = [] # Data msg type names, should be included in the message
self.archNode: Dict[str, List[Transaction]] = {}
self.stableStates: Dict[str, List[str]] = {} # [arch_name, List[stable_state_names]
self.initStateNodes: Dict[str, str] = {} # This is missing
if file and filename:
self.filename = filename
lexer = ProtoCCLexer(antlr3.StringStream(file))
parser = ProtoCCParser(antlr3.CommonTokenStream(lexer))
tree = parser.document().getTree()
new_tree_base = copy_tree(tree)
self.pdebug(new_tree_base.toStringTree())
self._ParseNodes(new_tree_base)
self.perror("Accesses for SSP not defined", self.checkAccessBehaviourDefined())
self.perror("Terminal states detected in SSP", self.checkAllStatesReachable())
if graphexport:
self._dArch()
def testFillBuffer(self):
"""CommonTokenStream.fillBuffer()"""
self.source.tokens.append(
antlr3.CommonToken(type=12)
)
self.source.tokens.append(
antlr3.CommonToken(type=13)
)
self.source.tokens.append(
antlr3.CommonToken(type=14)
)
self.source.tokens.append(
antlr3.CommonToken(type=antlr3.EOF)
)
stream = antlr3.CommonTokenStream(self.source)
stream.fillBuffer()
self.failUnlessEqual(len(stream.tokens), 3)
self.failUnlessEqual(stream.tokens[0].type, 12)
self.failUnlessEqual(stream.tokens[1].type, 13)
self.failUnlessEqual(stream.tokens[2].type, 14)
def compile(self, srcfile, base_dir, output_dir):
#fp = codecs.open(sys.argv[1], 'r', 'utf-8')
fp = open(srcfile, 'r')
char_stream = antlr3.ANTLRInputStream(fp)
lexer = ExprLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = ExprParser(tokens)
r = parser.prog()
# this is the root of the AST
root = r.tree
#print (root.toStringTree())
#print '-------'
nodes = antlr3.tree.CommonTreeNodeStream(root)
nodes.setTokenStream(tokens)
from Eval import Eval
eval = Eval(nodes)
#######################################
head, tail = os.path.split(srcfile)
if not os.path.exists(output_dir):
os.mkdir(output_dir)
if not os.path.exists(output_dir + '/__init__.py'):
fp = open(output_dir + '/__init__.py', 'w')
fp.close()
dstfile = os.path.normpath(output_dir + '/' + tail.split('.')[0] + '.py')
#print 'compile: %-30s=> %s' % (srcfile, dstfile)
cpy = CpyBuilder(dstfile, base_dir, output_dir)
eval.prog(cpy)
return dstfile
def parse_str(self, content):
stream = antlr3.ANTLRInputStream(StringIO.StringIO(content))
lexer = CoreLexer(stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = CoreParser(tokens)
ast = parser.program()
return ast.tree
def CreateParser(parse_string):
"""Creates a Groc Parser."""
input_string = antlr3.ANTLRStringStream(parse_string)
lexer = GrocLexerWithErrors(input_string)
tokens = antlr3.CommonTokenStream(lexer)
parser = GrocParserWithErrors(tokens)
return parser
def execParser(self, grammar, grammarEntry, input, expectErrors=False):
lexerCls, parserCls = self.compileInlineGrammar(grammar)
cStream = antlr3.StringStream(input)
lexer = lexerCls(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = parserCls(tStream)
r = getattr(parser, grammarEntry)()
if not expectErrors:
self.assertEquals(len(parser._errors), 0, parser._errors)
result = ""
if r is not None:
if hasattr(r, 'result'):
result += r.result
if r.tree is not None:
result += r.tree.toStringTree()
if not expectErrors:
return result
else:
return result, parser._errors
def _token_stream(name, text):
""" Helper method for generating a token stream from text. """
char_stream = antlr3.ANTLRStringStream(text)
lexer = DemystifyLexer.DemystifyLexer(char_stream)
lexer.card = name
# tokenizes completely and logs on errors
return antlr3.CommonTokenStream(lexer)
def testLTEmptySource(self):
"""CommonTokenStream.LT(): EOF (empty source)"""
stream = antlr3.CommonTokenStream(self.source)
lt1 = stream.LT(1)
self.failUnlessEqual(lt1.type, antlr3.EOF)
def testSetTokenSource(self):
"""CommonTokenStream.setTokenSource()"""
stream = antlr3.CommonTokenStream(None)
stream.setTokenSource(self.source)
self.failUnlessEqual(stream.index(), -1)
self.failUnlessEqual(stream.channel, antlr3.DEFAULT_CHANNEL)
def testValid1(self):
cStream = antlr3.StringStream(' a')
lexer = self.getLexer(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = self.getParser(tStream)
events = parser.a()
def testValid(self):
cStream = antlr3.StringStream('foobar')
lexer = self.getLexer(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = self.getParser(tStream)
parser.document()
assert parser.events == ['before', 'after']
def testValid(self):
cStream = antlr3.StringStream("int foo;")
lexer = self.getLexer(cStream)
tStream = antlr3.CommonTokenStream(lexer)
parser = self.getParser(tStream)
parser.program()
assert len(parser.reportedErrors) == 0, parser.reportedErrors
def __init__(self, file=""):
if file:
self.file = file
lexer = ProtoCCLexer(antlr3.StringStream(open(file).read()))
parser = ProtoCCParser(antlr3.CommonTokenStream(lexer))
tree = parser.document().getTree()
pdebug(tree.toStringTree())
self._ParseNodes(tree)
