def parse(notation):
"""Parse a notation string.
:return: An ANTLR AST. Use one of the supplied visitors (or write your own)
to turn it into useful output.
"""
substituted = substitute(notation)
lexer = TacticNotationsLexer(InputStream(substituted))
return TacticNotationsParser(CommonTokenStream(lexer)).top()
def java_tokenize(line):
# stream = InputStream(line.decode('utf-8', 'ignore'))
stream = InputStream(line)
lexer = JavaLexer(stream)
tokens = CommonTokenStream(lexer)
tokens.fetch(100000)
# tokenlist=set([token.text for token in tokens.tokens])
token_list = list([token.text for token in tokens.tokens])
return token_list # tokens.tokens
def parse(self, equel):
lexer = equelLexer(equel)
tokenstream = CommonTokenStream(lexer)
parser = equelParser(tokenstream)
parsetree = parser.equel()
walker = ParseTreeWalker()
listener = EQUELParserListener(self.engine)
walker.walk(listener, parsetree)
return parsetree
def make_parser(data: str) -> RelayParser:
"""Construct a RelayParser a given data stream."""
input_stream = InputStream(data)
lexer = RelayLexer(input_stream)
lexer.addErrorListener(StrictErrorListener(data))
token_stream = CommonTokenStream(lexer)
p = RelayParser(token_stream)
p.addErrorListener(StrictErrorListener(data))
return p
def reversePython(self, umlFrame: UmlClassDiagramsFrame,
directoryName: str, files: List[str]):
"""
Reverse engineering Python files to OglClass's
Args:
umlFrame: The uml frame to display on
directoryName: The directory name where the selected files reside
files: A list of files to parse
"""
fileCount: int = len(files)
dlg = ProgressDialog('Parsing Files',
'Starting',
parent=umlFrame,
style=PD_APP_MODAL | PD_ELAPSED_TIME)
dlg.SetRange(fileCount)
currentFileCount: int = 0
onGoingParents: Parents = Parents({})
for fileName in files:
try:
fqFileName: str = f'{directoryName}{osSep}{fileName}'
self.logger.info(f'Processing file: {fqFileName}')
dlg.Update(currentFileCount, f'Processing: {fileName}')
fileStream: FileStream = FileStream(fqFileName)
lexer: Python3Lexer = Python3Lexer(fileStream)
stream: CommonTokenStream = CommonTokenStream(lexer)
parser: Python3Parser = Python3Parser(stream)
tree: Python3Parser.File_inputContext = parser.file_input()
if parser.getNumberOfSyntaxErrors() != 0:
self.logger.error(
f"File {fileName} contains {parser.getNumberOfSyntaxErrors()} syntax errors"
)
# TODO: Put up a dialog
continue
self.visitor = PyutPythonVisitor()
self.visitor.parents = onGoingParents
self.visitor.visit(tree)
self._generatePyutClasses()
onGoingParents = self.visitor.parents
currentFileCount += 1
except (ValueError, Exception) as e:
from org.pyut.errorcontroller.ErrorManager import ErrorManager
eMsg: str = f'file: {fileName}\n{e} - {ErrorManager.getErrorInfo()}'
self.logger.error(eMsg)
dlg.Destroy()
raise PythonParseException(eMsg)
dlg.Destroy()
self._generateOglClasses(umlFrame)
self._layoutUmlClasses(umlFrame)
self._generateInheritanceLinks(umlFrame)
def read(self, pgm_file):
pgm_stream = FileStream(pgm_file)
lexer = ArithLangLexer(pgm_stream)
stream = CommonTokenStream(lexer)
parser = ArithLangParser(stream)
pgm = parser.program()
# Uncomment the following line to print the parse tree.
# print(pgm.toStringTree(recog=parser))
return pgm.ast
def get_definitions(file_path):
input_stream = FileStream(file_path, encoding='utf-8')
lexer = Python3Lexer(input_stream)
stream = CommonTokenStream(lexer)
parser = Python3Parser(stream)
tree = parser.file_input()
visitor = DefinitionsVisitor()
return visitor.visit(tree)
def main(argv):
dir_app = os.path.dirname(os.path.abspath(__file__))
dir_tmp = os.path.join(dir_app, 'tmp')
if len(sys.argv) > 2:
print('usage: ./run_data_extraction.py [pdf-directory-abs]')
sys.exit(1)
elif len(sys.argv) == 2:
dir_pdf = sys.argv[1]
else:
dir_pdf = os.path.join(dir_app, 'pdf')
# get all the PDF filenames
pdfFiles = []
for filename in os.listdir(dir_pdf):
if filename.endswith('.pdf'):
pdfFiles.append(filename)
pdfFiles.sort(key=str.lower)
print(pdfFiles)
# initiate the database class and open a database connection
db = Database()
listen_div = ListenerDividend()
for filename in pdfFiles:
# read pdf file, convert it to text, and store it as text file in the
# tmp directory for further processing/debugging
print('processing file ' + filename)
tmp_filename = os.path.join(dir_tmp, filename.replace('.pdf', '.txt'))
sometext = convert_pdf_to_txt(os.path.join(dir_pdf, filename))
f = open(tmp_filename, mode='w', encoding='UTF-8')
f.write(sometext)
f.close()
istream = FileStream(tmp_filename, encoding='UTF-8')
# parse the file
lexer = PdfDividendLexer(istream)
stream = CommonTokenStream(lexer)
parser = PdfDividendParser(stream)
tree = parser.rules()
#print(tree.toStringTree(recog=parser))
# listen to the words
walker = ParseTreeWalker()
walker.walk(listen_div, tree)
# write listen words and filename to database
db.write_to_database(listen_div, filename)
# clean up tmp file
os.remove(tmp_filename)
listen_div.reset()
# close database connection
db.close()
def test(input, expect, num):
if isinstance(input, str):
inputfile = TestUtil.makeSource(input, num)
lexer = Lexer(inputfile)
tokens = CommonTokenStream(lexer)
try:
listener = TestParser.createErrorListener()
parser = Parser(tokens)
parser.removeErrorListeners()
parser.addErrorListener(listener)
tree = parser.program()
except SyntaxException as f:
if VERBOSE_MODE:
msg = f.message.split(':')[0].split(' ')
line = int(msg[3])
col = int(msg[5])
error_line = open(TEST_DIR + '/' + str(num) +
'.txt').read()
error_line = error_line.split('\n')[line - 1]
print('[*] {}'.format(num))
print(error_line)
print('~' * (col) + '^')
print(f.message)
raise f
asttree = ASTGeneration().visit(tree)
else:
inputfile = TestUtil.makeSource(str(input), num)
asttree = input
# checker = StaticChecker(asttree)
# try:
# checker.check()
# except StaticError as e:
# if VERBOSE_MODE:
# print('[*] {} [{}]\n-->[{}]'.format(
# num, inputfile, str(asttree)))
# print('*' * 10, '{}'.format(num), str(e))
# raise e
TestCodeGen.check(SOL_DIR, asttree, num)
dest = open(SOL_DIR + "/" + str(num) + ".txt", "r")
line = dest.read()
if VERBOSE_MODE:
err = '*' if line != expect else '+'
print('''[{}] {} [{}]
-->[{}]
-->[
{}
]
-->[{}]
===[{}]'''.format(err, num, input, str(asttree),
open(SOL_DIR + '/' + str(num) + '/' + 'MPClass.j').read(),
line, expect))
print('----------')
return line == expect
def __init__(self, parse_java_file_path):
super().__init__()
self.javafile_path = parse_java_file_path
self.parser = Java8Parser(
CommonTokenStream(
Java8Lexer(FileStream(parse_java_file_path,
encoding="utf-8"))))
self.insert_points = []
self.insert_tokens = []
self.testname_re = re.compile(r"test[0-9]+")
def translate(formula1, formula2):
'''TODO:
'''
lexer = stlLexer(InputStream(formula1))
tokens = CommonTokenStream(lexer)
parser = stlParser(tokens)
t = parser.stlProperty()
ast1 = STLAbstractSyntaxTreeExtractor().visit(t)
print 'AST 1:', ast1
pnf1 = ast1.pnf()
print 'PNF 1:', pnf1
lexer = stlLexer(InputStream(formula2))
tokens = CommonTokenStream(lexer)
parser = stlParser(tokens)
t = parser.stlProperty()
ast2 = STLAbstractSyntaxTreeExtractor().visit(t)
print 'AST 2:', ast2
pnf2 = ast2.pnf()
print 'PNF 2:', pnf2
variables = list(pnf1.variables() | pnf2.variables())
formula1_constr, formula1, count = code_from_stl(pnf1, variables, 0)
formula2_constr, formula2, count = code_from_stl(pnf2, variables, count)
### HACK: multiply by 10
bound = int(round(10 * max(pnf1.bound(), pnf2.bound())))
program_12 = program_string.format(bound=bound,
no_variables=len(variables),
formula1_construction=formula1_constr,
formula2_construction=formula2_constr,
formula1=formula1,
formula2=formula2)
program_21 = program_string.format(bound=bound,
no_variables=len(variables),
formula1_construction=formula1_constr,
formula2_construction=formula2_constr,
formula1=formula2,
formula2=formula1)
return program_12, program_21
def print_tokens(srcfile):
from antlr4 import FileStream, CommonTokenStream
from dynamod.parser.DynamodLexer import DynamodLexer
input = FileStream(srcfile)
lexer = DynamodLexer(input)
stream = CommonTokenStream(lexer)
stream.fill()
for token in stream.getTokens(0, 9999999):
print(str(token))
def test_bad_tnsnames_file(self):
tnsnames_file = '{0}/testFiles/tnsnames_false.ora'.format(
self._script_path)
input_file_stream = FileStream(tnsnames_file)
lexer = tnsnamesLexer(input_file_stream)
stream = CommonTokenStream(lexer)
parser = TnsNamesParserWithException(stream)
with self.assertRaisesRegexp(Exception, "Syntax error") as cm:
tree = parser.tnsnames()
def parse_one(problem):
lexer = tptp_v7_0_0_0Lexer(FileStream(problem.file))
stream = CommonTokenStream(lexer)
parser = tptp_v7_0_0_0Parser(stream)
tree = parser.tptp_file()
listener = QuantifierFeaturesListener()
walker = ParseTreeWalker()
walker.walk(listener, tree)
return listener
def java2tree(line):
# stream = InputStream(line.decode('utf-8', 'ignore'))
stream = InputStream(line)
lexer = JavaLexer(stream)
toks = CommonTokenStream(lexer)
parser = JavaParser(toks)
tree = parser.compilationUnit()
return tree, toks
def main(argv):
input_ = FileStream(argv[1])
lexer = ChatLexer(input_)
parser = ChatParser(CommonTokenStream(lexer))
tree = parser.chat()
with open("output.html", "w") as output:
chat_listener = HtmlChatListener(output)
walker = ParseTreeWalker()
walker.walk(chat_listener, tree)
def loads(s):
inp = InputStream(s)
lexer = renLexer(inp)
lexer._listeners = [RenErrorListener()]
stream = CommonTokenStream(lexer)
parser = renParser(stream)
parser._listeners = [RenErrorListener()]
tree = parser.root()
visitor = Visitor()
return visitor.visit(tree)
def parseString(s, entry, srcName='',
ParserClass=VerexParser,
LexerClass=VerexLexer):
istr = createInputStream(fromstr=s, srcName=srcName)
parser = parserclass(
CommonTokenStream(
LexerClass(istr)
))
parser.buildParseTrees = True
return getattr(parser, entry)()
def parseFile(filename, entry=VerexParser.vfile,
ParserClass=VerexParser,
LexerClass=VerexLexer):
istr = createInputStream(filename=filename)
parser = parserclass(
CommonTokenStream(
LexerClass(istr)
))
parser.buildParseTrees = True
return getattr(parser, entry)()
def main(argv):
input_stream = FileStream(argv[1])
lexer = HelloLexer(input_stream)
stream = CommonTokenStream(lexer)
parser = HelloParser(stream)
tree = parser.r()
listener = TestListener()
walker = ParseTreeWalker()
walker.walk(listener, tree)
def Lexer(inputStream):
lexer = MiniDecafExprLexer(inputStream)
class BailErrorListener:
def syntaxError(self, recognizer, offendingSymbol, line, column, msg,
e):
raise MiniDecafError(f"lexer error at {line},{column}")
lexer.addErrorListener(BailErrorListener())
return CommonTokenStream(lexer)
def __init__(self, code):
if isinstance(code, str):
self.stream = InputStream(code)
else:
self.stream = code
self.lexer = SolidityLexer(self.stream)
self.tokens = CommonTokenStream(self.lexer)
self.parser = SolidityParser(self.tokens)
self.parser._listeners = [MyErrorListener(code)]
self.tree = self.parser.sourceUnit()
def __init__(self, sql):
self._tokens = {}
self.sql = sql
self.input_stream = CaseChangingCharInputStream(self.sql)
self.lexer = MySqlLexer(self.input_stream)
self.token_stream = CommonTokenStream(self.lexer)
self.parser = MySqlParser(self.token_stream)
self.tree = self.parser.root()
self.printer = MyListener(self._tokens)
self.walker = ParseTreeWalker()
def fromString(s):
ins = InputStream(s)
lexer = PADLexer(ins)
stream = CommonTokenStream(lexer)
parser = PADParser(stream)
tree = parser.formula()
builder = FormulaBuilder()
walker = ParseTreeWalker()
walker.walk(builder, tree)
return builder.getFormula()
def parse(self, path, rel_path):
package = ''
if '/' in rel_path:
package = rel_path[:rel_path.rfind('/')].replace('/', '.')
package = SymbolTable().enter_package(package)
src = SourceFile(path)
parser = PlayParser(
CommonTokenStream(PlayLexer(FileStream(path, encoding='utf-8'))))
Option().nodes[src] = parser.compilationUnit()
package.source_files.append(src)
def setupParseLog(self):
self.lexer = importlib.import_module('DataParser.parser.Lexer').Lexer(
self.file_stream)
self.stream = CommonTokenStream(self.lexer)
self.parser = importlib.import_module(
'DataParser.parser.Lexer').Parser(self.stream)
self.printer = importlib.import_module(
'DataParser.parser.listner_for_overriding').Listener(
data_structure=self)
pass
def fromFile(fname):
ins = FileStream(fname)
lexer = PADLexer(ins)
stream = CommonTokenStream(lexer)
parser = PADParser(stream)
tree = parser.formula()
builder = FormulaBuilder()
walker = ParseTreeWalker()
walker.walk(builder, tree)
return builder.getFormula()
def check_astgen(self, input, expect):
lexer = BKITLexer(InputStream(input))
tokens = CommonTokenStream(lexer)
parser = BKITParser(tokens)
# parser.setTrace(True)
tree = parser.program()
asttree = ASTGeneration().visit(tree)
self.assertEqual(
str(asttree), str(expect),
"\n" + pformat(asdict(asttree))) # +"\n"+pformat(asdict(expect)))
def parse(self, fname: str):
input_stream = FileStream(fname)
lexer = DiagnosisLexer(input_stream)
stream = CommonTokenStream(lexer)
parser = DiagnosisParser(stream)
listener = DiagnosisListener()
parser.addErrorListener(listener)
tree = parser.system_verilog_text()
return listener.errors
def split(self, path: str):
input = InputStream(path)
lexer = XPathLexer(input)
def recover(self, e):
raise e
lexer.recover = recover
lexer.removeErrorListeners()
lexer.addErrorListener(
ErrorListener()) # XPathErrorListener does no more
tokenStream = CommonTokenStream(lexer)
try:
tokenStream.fill()
except LexerNoViableAltException as e:
pos = lexer.column
msg = "Invalid tokens or characters at index %d in path '%s'" % (
pos, path)
raise Exception(msg, e)
tokens = iter(tokenStream.tokens)
elements = list()
for el in tokens:
invert = False
anywhere = False
# Check for path separators, if none assume root
if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]:
anywhere = el.type == XPathLexer.ANYWHERE
next_el = next(tokens, None)
if not next_el:
raise Exception('Missing element after %s' % el.getText())
else:
el = next_el
# Check for bangs
if el.type == XPathLexer.BANG:
invert = True
next_el = next(tokens, None)
if not next_el:
raise Exception('Missing element after %s' % el.getText())
else:
el = next_el
# Add searched element
if el.type in [
XPathLexer.TOKEN_REF, XPathLexer.RULE_REF,
XPathLexer.WILDCARD, XPathLexer.STRING
]:
element = self.getXPathElement(el, anywhere)
element.invert = invert
elements.append(element)
elif el.type == Token.EOF:
break
else:
raise Exception("Unknown path element %s" %
lexer.symbolicNames[el.type])
return elements
