def parse(notation): """Parse a notation string. :return: An ANTLR AST. Use one of the supplied visitors (or write your own) to turn it into useful output. """ substituted = substitute(notation) lexer = TacticNotationsLexer(InputStream(substituted)) return TacticNotationsParser(CommonTokenStream(lexer)).top()
def java_tokenize(line): # stream = InputStream(line.decode('utf-8', 'ignore')) stream = InputStream(line) lexer = JavaLexer(stream) tokens = CommonTokenStream(lexer) tokens.fetch(100000) # tokenlist=set([token.text for token in tokens.tokens]) token_list = list([token.text for token in tokens.tokens]) return token_list # tokens.tokens
def parse(self, equel): lexer = equelLexer(equel) tokenstream = CommonTokenStream(lexer) parser = equelParser(tokenstream) parsetree = parser.equel() walker = ParseTreeWalker() listener = EQUELParserListener(self.engine) walker.walk(listener, parsetree) return parsetree
def make_parser(data: str) -> RelayParser: """Construct a RelayParser a given data stream.""" input_stream = InputStream(data) lexer = RelayLexer(input_stream) lexer.addErrorListener(StrictErrorListener(data)) token_stream = CommonTokenStream(lexer) p = RelayParser(token_stream) p.addErrorListener(StrictErrorListener(data)) return p
def reversePython(self, umlFrame: UmlClassDiagramsFrame, directoryName: str, files: List[str]): """ Reverse engineering Python files to OglClass's Args: umlFrame: The uml frame to display on directoryName: The directory name where the selected files reside files: A list of files to parse """ fileCount: int = len(files) dlg = ProgressDialog('Parsing Files', 'Starting', parent=umlFrame, style=PD_APP_MODAL | PD_ELAPSED_TIME) dlg.SetRange(fileCount) currentFileCount: int = 0 onGoingParents: Parents = Parents({}) for fileName in files: try: fqFileName: str = f'{directoryName}{osSep}{fileName}' self.logger.info(f'Processing file: {fqFileName}') dlg.Update(currentFileCount, f'Processing: {fileName}') fileStream: FileStream = FileStream(fqFileName) lexer: Python3Lexer = Python3Lexer(fileStream) stream: CommonTokenStream = CommonTokenStream(lexer) parser: Python3Parser = Python3Parser(stream) tree: Python3Parser.File_inputContext = parser.file_input() if parser.getNumberOfSyntaxErrors() != 0: self.logger.error( f"File {fileName} contains {parser.getNumberOfSyntaxErrors()} syntax errors" ) # TODO: Put up a dialog continue self.visitor = PyutPythonVisitor() self.visitor.parents = onGoingParents self.visitor.visit(tree) self._generatePyutClasses() onGoingParents = self.visitor.parents currentFileCount += 1 except (ValueError, Exception) as e: from org.pyut.errorcontroller.ErrorManager import ErrorManager eMsg: str = f'file: {fileName}\n{e} - {ErrorManager.getErrorInfo()}' self.logger.error(eMsg) dlg.Destroy() raise PythonParseException(eMsg) dlg.Destroy() self._generateOglClasses(umlFrame) self._layoutUmlClasses(umlFrame) self._generateInheritanceLinks(umlFrame)
def read(self, pgm_file): pgm_stream = FileStream(pgm_file) lexer = ArithLangLexer(pgm_stream) stream = CommonTokenStream(lexer) parser = ArithLangParser(stream) pgm = parser.program() # Uncomment the following line to print the parse tree. # print(pgm.toStringTree(recog=parser)) return pgm.ast
def get_definitions(file_path): input_stream = FileStream(file_path, encoding='utf-8') lexer = Python3Lexer(input_stream) stream = CommonTokenStream(lexer) parser = Python3Parser(stream) tree = parser.file_input() visitor = DefinitionsVisitor() return visitor.visit(tree)
def main(argv): dir_app = os.path.dirname(os.path.abspath(__file__)) dir_tmp = os.path.join(dir_app, 'tmp') if len(sys.argv) > 2: print('usage: ./run_data_extraction.py [pdf-directory-abs]') sys.exit(1) elif len(sys.argv) == 2: dir_pdf = sys.argv[1] else: dir_pdf = os.path.join(dir_app, 'pdf') # get all the PDF filenames pdfFiles = [] for filename in os.listdir(dir_pdf): if filename.endswith('.pdf'): pdfFiles.append(filename) pdfFiles.sort(key=str.lower) print(pdfFiles) # initiate the database class and open a database connection db = Database() listen_div = ListenerDividend() for filename in pdfFiles: # read pdf file, convert it to text, and store it as text file in the # tmp directory for further processing/debugging print('processing file ' + filename) tmp_filename = os.path.join(dir_tmp, filename.replace('.pdf', '.txt')) sometext = convert_pdf_to_txt(os.path.join(dir_pdf, filename)) f = open(tmp_filename, mode='w', encoding='UTF-8') f.write(sometext) f.close() istream = FileStream(tmp_filename, encoding='UTF-8') # parse the file lexer = PdfDividendLexer(istream) stream = CommonTokenStream(lexer) parser = PdfDividendParser(stream) tree = parser.rules() #print(tree.toStringTree(recog=parser)) # listen to the words walker = ParseTreeWalker() walker.walk(listen_div, tree) # write listen words and filename to database db.write_to_database(listen_div, filename) # clean up tmp file os.remove(tmp_filename) listen_div.reset() # close database connection db.close()
def test(input, expect, num): if isinstance(input, str): inputfile = TestUtil.makeSource(input, num) lexer = Lexer(inputfile) tokens = CommonTokenStream(lexer) try: listener = TestParser.createErrorListener() parser = Parser(tokens) parser.removeErrorListeners() parser.addErrorListener(listener) tree = parser.program() except SyntaxException as f: if VERBOSE_MODE: msg = f.message.split(':')[0].split(' ') line = int(msg[3]) col = int(msg[5]) error_line = open(TEST_DIR + '/' + str(num) + '.txt').read() error_line = error_line.split('\n')[line - 1] print('[*] {}'.format(num)) print(error_line) print('~' * (col) + '^') print(f.message) raise f asttree = ASTGeneration().visit(tree) else: inputfile = TestUtil.makeSource(str(input), num) asttree = input # checker = StaticChecker(asttree) # try: # checker.check() # except StaticError as e: # if VERBOSE_MODE: # print('[*] {} [{}]\n-->[{}]'.format( # num, inputfile, str(asttree))) # print('*' * 10, '{}'.format(num), str(e)) # raise e TestCodeGen.check(SOL_DIR, asttree, num) dest = open(SOL_DIR + "/" + str(num) + ".txt", "r") line = dest.read() if VERBOSE_MODE: err = '*' if line != expect else '+' print('''[{}] {} [{}] -->[{}] -->[ {} ] -->[{}] ===[{}]'''.format(err, num, input, str(asttree), open(SOL_DIR + '/' + str(num) + '/' + 'MPClass.j').read(), line, expect)) print('----------') return line == expect
def __init__(self, parse_java_file_path): super().__init__() self.javafile_path = parse_java_file_path self.parser = Java8Parser( CommonTokenStream( Java8Lexer(FileStream(parse_java_file_path, encoding="utf-8")))) self.insert_points = [] self.insert_tokens = [] self.testname_re = re.compile(r"test[0-9]+")
def translate(formula1, formula2): '''TODO: ''' lexer = stlLexer(InputStream(formula1)) tokens = CommonTokenStream(lexer) parser = stlParser(tokens) t = parser.stlProperty() ast1 = STLAbstractSyntaxTreeExtractor().visit(t) print 'AST 1:', ast1 pnf1 = ast1.pnf() print 'PNF 1:', pnf1 lexer = stlLexer(InputStream(formula2)) tokens = CommonTokenStream(lexer) parser = stlParser(tokens) t = parser.stlProperty() ast2 = STLAbstractSyntaxTreeExtractor().visit(t) print 'AST 2:', ast2 pnf2 = ast2.pnf() print 'PNF 2:', pnf2 variables = list(pnf1.variables() | pnf2.variables()) formula1_constr, formula1, count = code_from_stl(pnf1, variables, 0) formula2_constr, formula2, count = code_from_stl(pnf2, variables, count) ### HACK: multiply by 10 bound = int(round(10 * max(pnf1.bound(), pnf2.bound()))) program_12 = program_string.format(bound=bound, no_variables=len(variables), formula1_construction=formula1_constr, formula2_construction=formula2_constr, formula1=formula1, formula2=formula2) program_21 = program_string.format(bound=bound, no_variables=len(variables), formula1_construction=formula1_constr, formula2_construction=formula2_constr, formula1=formula2, formula2=formula1) return program_12, program_21
def print_tokens(srcfile): from antlr4 import FileStream, CommonTokenStream from dynamod.parser.DynamodLexer import DynamodLexer input = FileStream(srcfile) lexer = DynamodLexer(input) stream = CommonTokenStream(lexer) stream.fill() for token in stream.getTokens(0, 9999999): print(str(token))
def test_bad_tnsnames_file(self): tnsnames_file = '{0}/testFiles/tnsnames_false.ora'.format( self._script_path) input_file_stream = FileStream(tnsnames_file) lexer = tnsnamesLexer(input_file_stream) stream = CommonTokenStream(lexer) parser = TnsNamesParserWithException(stream) with self.assertRaisesRegexp(Exception, "Syntax error") as cm: tree = parser.tnsnames()
def parse_one(problem): lexer = tptp_v7_0_0_0Lexer(FileStream(problem.file)) stream = CommonTokenStream(lexer) parser = tptp_v7_0_0_0Parser(stream) tree = parser.tptp_file() listener = QuantifierFeaturesListener() walker = ParseTreeWalker() walker.walk(listener, tree) return listener
def java2tree(line): # stream = InputStream(line.decode('utf-8', 'ignore')) stream = InputStream(line) lexer = JavaLexer(stream) toks = CommonTokenStream(lexer) parser = JavaParser(toks) tree = parser.compilationUnit() return tree, toks
def main(argv): input_ = FileStream(argv[1]) lexer = ChatLexer(input_) parser = ChatParser(CommonTokenStream(lexer)) tree = parser.chat() with open("output.html", "w") as output: chat_listener = HtmlChatListener(output) walker = ParseTreeWalker() walker.walk(chat_listener, tree)
def loads(s): inp = InputStream(s) lexer = renLexer(inp) lexer._listeners = [RenErrorListener()] stream = CommonTokenStream(lexer) parser = renParser(stream) parser._listeners = [RenErrorListener()] tree = parser.root() visitor = Visitor() return visitor.visit(tree)
def parseString(s, entry, srcName='', ParserClass=VerexParser, LexerClass=VerexLexer): istr = createInputStream(fromstr=s, srcName=srcName) parser = parserclass( CommonTokenStream( LexerClass(istr) )) parser.buildParseTrees = True return getattr(parser, entry)()
def parseFile(filename, entry=VerexParser.vfile, ParserClass=VerexParser, LexerClass=VerexLexer): istr = createInputStream(filename=filename) parser = parserclass( CommonTokenStream( LexerClass(istr) )) parser.buildParseTrees = True return getattr(parser, entry)()
def main(argv): input_stream = FileStream(argv[1]) lexer = HelloLexer(input_stream) stream = CommonTokenStream(lexer) parser = HelloParser(stream) tree = parser.r() listener = TestListener() walker = ParseTreeWalker() walker.walk(listener, tree)
def Lexer(inputStream): lexer = MiniDecafExprLexer(inputStream) class BailErrorListener: def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): raise MiniDecafError(f"lexer error at {line},{column}") lexer.addErrorListener(BailErrorListener()) return CommonTokenStream(lexer)
def __init__(self, code): if isinstance(code, str): self.stream = InputStream(code) else: self.stream = code self.lexer = SolidityLexer(self.stream) self.tokens = CommonTokenStream(self.lexer) self.parser = SolidityParser(self.tokens) self.parser._listeners = [MyErrorListener(code)] self.tree = self.parser.sourceUnit()
def __init__(self, sql): self._tokens = {} self.sql = sql self.input_stream = CaseChangingCharInputStream(self.sql) self.lexer = MySqlLexer(self.input_stream) self.token_stream = CommonTokenStream(self.lexer) self.parser = MySqlParser(self.token_stream) self.tree = self.parser.root() self.printer = MyListener(self._tokens) self.walker = ParseTreeWalker()
def fromString(s): ins = InputStream(s) lexer = PADLexer(ins) stream = CommonTokenStream(lexer) parser = PADParser(stream) tree = parser.formula() builder = FormulaBuilder() walker = ParseTreeWalker() walker.walk(builder, tree) return builder.getFormula()
def parse(self, path, rel_path): package = '' if '/' in rel_path: package = rel_path[:rel_path.rfind('/')].replace('/', '.') package = SymbolTable().enter_package(package) src = SourceFile(path) parser = PlayParser( CommonTokenStream(PlayLexer(FileStream(path, encoding='utf-8')))) Option().nodes[src] = parser.compilationUnit() package.source_files.append(src)
def setupParseLog(self): self.lexer = importlib.import_module('DataParser.parser.Lexer').Lexer( self.file_stream) self.stream = CommonTokenStream(self.lexer) self.parser = importlib.import_module( 'DataParser.parser.Lexer').Parser(self.stream) self.printer = importlib.import_module( 'DataParser.parser.listner_for_overriding').Listener( data_structure=self) pass
def fromFile(fname): ins = FileStream(fname) lexer = PADLexer(ins) stream = CommonTokenStream(lexer) parser = PADParser(stream) tree = parser.formula() builder = FormulaBuilder() walker = ParseTreeWalker() walker.walk(builder, tree) return builder.getFormula()
def check_astgen(self, input, expect): lexer = BKITLexer(InputStream(input)) tokens = CommonTokenStream(lexer) parser = BKITParser(tokens) # parser.setTrace(True) tree = parser.program() asttree = ASTGeneration().visit(tree) self.assertEqual( str(asttree), str(expect), "\n" + pformat(asdict(asttree))) # +"\n"+pformat(asdict(expect)))
def parse(self, fname: str): input_stream = FileStream(fname) lexer = DiagnosisLexer(input_stream) stream = CommonTokenStream(lexer) parser = DiagnosisParser(stream) listener = DiagnosisListener() parser.addErrorListener(listener) tree = parser.system_verilog_text() return listener.errors
def split(self, path: str): input = InputStream(path) lexer = XPathLexer(input) def recover(self, e): raise e lexer.recover = recover lexer.removeErrorListeners() lexer.addErrorListener( ErrorListener()) # XPathErrorListener does no more tokenStream = CommonTokenStream(lexer) try: tokenStream.fill() except LexerNoViableAltException as e: pos = lexer.column msg = "Invalid tokens or characters at index %d in path '%s'" % ( pos, path) raise Exception(msg, e) tokens = iter(tokenStream.tokens) elements = list() for el in tokens: invert = False anywhere = False # Check for path separators, if none assume root if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]: anywhere = el.type == XPathLexer.ANYWHERE next_el = next(tokens, None) if not next_el: raise Exception('Missing element after %s' % el.getText()) else: el = next_el # Check for bangs if el.type == XPathLexer.BANG: invert = True next_el = next(tokens, None) if not next_el: raise Exception('Missing element after %s' % el.getText()) else: el = next_el # Add searched element if el.type in [ XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD, XPathLexer.STRING ]: element = self.getXPathElement(el, anywhere) element.invert = invert elements.append(element) elif el.type == Token.EOF: break else: raise Exception("Unknown path element %s" % lexer.symbolicNames[el.type]) return elements