def main(inputname, verbose): if inputname: lexer = SujetALexer(FileStream(inputname)) else: lexer = SujetALexer(StdinStream()) if not verbose: lexer.removeErrorListener(ConsoleErrorListener.INSTANCE) counter = CountErrorListener() lexer._listeners.append(counter) stream = CommonTokenStream(lexer) stream.fill() if counter.count > 0: # wrong token ! print("lexical error") exit(1) parser = SujetAParser(stream) if not verbose: parser.removeErrorListener(ConsoleErrorListener.INSTANCE) parser._listeners.append(counter) parser.start() if counter.count > 0: print("syntax error") exit(2) else: print("ok") exit(0)
def parse(input_: Union[str, FileStream], default_base: Optional[str]=None) -> Optional[Schema]: """ Parse the text in infile and return the resulting schema :param input_: text or input stream to parse :param default_base_: base URI for relative URI's in schema :return: ShExJ Schema object. None if error. """ # Step 1: Tokenize the input stream error_listener = ParseErrorListener() if not isinstance(input_, FileStream): input_ = InputStream(input_) lexer = ShExDocLexer(input_) lexer.addErrorListener(error_listener) tokens = CommonTokenStream(lexer) tokens.fill() if error_listener.n_errors: # Lexer prints errors directly return None # Step 2: Generate the parse tree parser = ShExDocParser(tokens) parser.addErrorListener(error_listener) parse_tree = parser.shExDoc() if error_listener.n_errors: print('\n'.join(error_listener.errors), file=sys.stderr) return None # Step 3: Transform the results the results parser = ShexDocParser(default_base=default_base) parser.visit(parse_tree) return parser.context.schema
def parse(input_: Union[str, InputStream], default_base: Optional[str] = None) -> Optional[Schema]: """ Parse the text in infile and return the resulting schema :param input_: text or input stream to parse :param default_base: base URI for relative URI's in schema :return: ShExJ Schema object. None if error. """ # Step 1: Tokenize the input stream error_listener = ParseErrorListener() if not isinstance(input_, InputStream): input_ = InputStream(input_) lexer = ShExDocLexer(input_) lexer.addErrorListener(error_listener) tokens = CommonTokenStream(lexer) tokens.fill() if error_listener.n_errors: # Lexer prints errors directly return None # Step 2: Generate the parse tree parser = ShExDocParser(tokens) parser.addErrorListener(error_listener) parse_tree = parser.shExDoc() if error_listener.n_errors: print('\n'.join(error_listener.errors), file=sys.stderr) return None # Step 3: Transform the results the results parser = ShexDocParser(default_base=default_base) parser.visit(parse_tree) return parser.context.schema
def tokenize(string: str) -> Tuple[ASTBuilderVisitor, PyNestMLParser]: lexer = PyNestMLLexer(InputStream(string)) # create a token stream stream = CommonTokenStream(lexer) stream.fill() parser = PyNestMLParser(stream) builder = ASTBuilderVisitor(stream.tokens) return builder, parser
def create_token_stream(self, line): istream = InputStream(line) lexer = MetrinkFrontendLexer(istream) stream = CommonTokenStream(lexer) stream.fill() return stream
def lex(i_stream): """Makes lexical analysis Returns: stream of tokens """ lexer = Python3Lexer(i_stream) t_stream = CommonTokenStream(lexer) t_stream.fill() return t_stream
def print_tokens(srcfile): from antlr4 import FileStream, CommonTokenStream from dynamod.parser.DynamodLexer import DynamodLexer input = FileStream(srcfile) lexer = DynamodLexer(input) stream = CommonTokenStream(lexer) stream.fill() for token in stream.getTokens(0, 9999999): print(str(token))
def split(self, path: str): input = InputStream(path) lexer = XPathLexer(input) def recover(self, e): raise e lexer.recover = recover lexer.removeErrorListeners() lexer.addErrorListener( ErrorListener()) # XPathErrorListener does no more tokenStream = CommonTokenStream(lexer) try: tokenStream.fill() except LexerNoViableAltException as e: pos = lexer.column msg = "Invalid tokens or characters at index %d in path '%s'" % ( pos, path) raise Exception(msg, e) tokens = iter(tokenStream.tokens) elements = list() for el in tokens: invert = False anywhere = False # Check for path separators, if none assume root if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]: anywhere = el.type == XPathLexer.ANYWHERE next_el = next(tokens, None) if not next_el: raise Exception('Missing element after %s' % el.getText()) else: el = next_el # Check for bangs if el.type == XPathLexer.BANG: invert = True next_el = next(tokens, None) if not next_el: raise Exception('Missing element after %s' % el.getText()) else: el = next_el # Add searched element if el.type in [ XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD, XPathLexer.STRING ]: element = self.getXPathElement(el, anywhere) element.invert = invert elements.append(element) elif el.type == Token.EOF: break else: raise Exception("Unknown path element %s" % lexer.symbolicNames[el.type]) return elements
def split(self, path: str): input = InputStream(path) lexer = XPathLexer(input) def recover(self, e): raise e lexer.recover = recover lexer.removeErrorListeners() lexer.addErrorListener( ErrorListener()) # XPathErrorListener does no more tokenStream = CommonTokenStream(lexer) try: tokenStream.fill() except LexerNoViableAltException as e: pos = lexer.getColumn() msg = "Invalid tokens or characters at index " + str( pos) + " in path '" + path + "'" raise Exception(msg, e) tokens = tokenStream.getTokens() elements = list() n = len(tokens) i = 0 while i < n: el = tokens[i] next = None if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]: anywhere = el.type == XPathLexer.ANYWHERE i += 1 next = tokens[i] invert = next.type == XPathLexer.BANG if invert: i += 1 next = tokens[i] pathElement = self.getXPathElement(next, anywhere) pathElement.invert = invert elements.append(pathElement) i += 1 elif el.type in [ XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD ]: elements.append(self.getXPathElement(el, False)) i += 1 elif el.type == Token.EOF: break else: raise Exception("Unknown path element " + str(el)) return elements
def get_stream(Lexer, file_path, strip_eof=True): inp = FileStream(file_path) lexer = Lexer(inp) stream = CommonTokenStream(lexer) stream.fill() tokens = stream.tokens if strip_eof: new_stream = [t.type for t in tokens[:-1]] else: new_stream = [t.type for t in tokens] return new_stream, tokens
def main(): # InputStream reads characters (from stdin in our case) input_stream = InputStream(sys.stdin.read()) # The generated lexer groups characters into Tokens ... lexer = Example1(input_stream) # ... and the stream of Tokens is managed by the TokenStream. stream = CommonTokenStream(lexer) # Display the token stream stream.fill() # needed to get stream.tokens (otherwise lazily filled-in) for t in stream.tokens: print(t) print("Finished")
def split(self, path: str): input = InputStream(path) lexer = XPathLexer(input) def recover(self, e): raise e lexer.recover = recover lexer.removeErrorListeners() lexer.addErrorListener(ErrorListener()) # XPathErrorListener does no more tokenStream = CommonTokenStream(lexer) try: tokenStream.fill() except LexerNoViableAltException as e: pos = lexer.getColumn() msg = "Invalid tokens or characters at index " + str(pos) + " in path '" + path + "'" raise Exception(msg, e) tokens = tokenStream.getTokens() elements = list() n = len(tokens) i = 0 while i < n: el = tokens[i] next = None if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]: anywhere = el.type == XPathLexer.ANYWHERE i += 1 next = tokens[i] invert = next.type == XPathLexer.BANG if invert: i += 1 next = tokens[i] pathElement = self.getXPathElement(next, anywhere) pathElement.invert = invert elements.append(pathElement) i += 1 elif el.type in [XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD]: elements.append(self.getXPathElement(el, False)) i += 1 elif el.type == Token.EOF: break else: raise Exception("Unknown path element " + str(el)) return elements
def parse(text: str, production_rule: str, listener) -> Optional[jsgParserVisitor]: """ Parse text fragment according to supplied production rule and evaluate with listener class. Example: parse("{1,*}", "ebnfSuffix", JSGEbnf) """ error_listener = ParseErrorListener() lexer = jsgLexer(InputStream(text)) lexer.addErrorListener(error_listener) tokens = CommonTokenStream(lexer) tokens.fill() if error_listener.n_errors: return None parser = jsgParser(tokens) parser.addErrorListener(error_listener) base_node = getattr(parser, production_rule)() listener_module = listener(JSGDocContext()) listener_module.visit(base_node) return listener_module if not error_listener.n_errors else None
def parse(input_: Union[str, FileStream], source: str, emit_header: bool=True) -> Optional[str]: """Parse the text in infile and save the results in outfile :param input_: string or stream to parse :param source: source name for python file header :param emit_header: True means include header in python file :return: python text if successful """ # Step 1: Tokenize the input stream error_listener = ParseErrorListener() if not isinstance(input_, FileStream): input_ = InputStream(input_) lexer = jsgLexer(input_) lexer.addErrorListener(error_listener) tokens = CommonTokenStream(lexer) tokens.fill() if error_listener.n_errors: return None # Step 2: Generate the parse tree parser = jsgParser(tokens) parser.addErrorListener(error_listener) parse_tree = parser.doc() if error_listener.n_errors: return None # Step 3: Transform the results the results parser = JSGDocParser() parser.visit(parse_tree) if parser.undefined_tokens(): for tkn in parser.undefined_tokens(): print("Undefined token: " + tkn) return None return parser.as_python(source, emit_header=emit_header)
import sys import argparse from antlr4 import FileStream, InputStream, CommonTokenStream from SimpleJavaLexer import SimpleJavaLexer from SimpleJavaParser import SimpleJavaParser from SimpleJavaErrorListener import VerboseListener if __name__ == '__main__': p = argparse.ArgumentParser() p.add_argument('-f', '--file', default=None) p.add_argument('-t', '--tokens', action="store_true") args = p.parse_args() if args.file: input_stream = FileStream(args.file, encoding='utf8') else: input_stream = InputStream(sys.stdin.read()) lexer = SimpleJavaLexer(input_stream) token_stream = CommonTokenStream(lexer) token_stream.fill() if args.tokens: for tok in token_stream.tokens: print(tok) parser = SimpleJavaParser(token_stream) parser.removeErrorListeners() parser.addErrorListener(VerboseListener()) parser.prog()
tree_representation += (" " * indent) + f"TOKEN='{tree.getText()}'\n" else: tree_representation += ( " " * indent) + f"{rule_names[tree.getRuleIndex()]}\n" for child in tree.children: generate_indented_tree(child, rule_names, indent + 1) if __name__ == "__main__": input_data = FileStream(sys.argv[1]) tokens_print = [] lexer = HelloLexer(input_data) stream = CommonTokenStream(lexer) stream.fill() for token in stream.tokens: if token.type != Token.EOF: token_attr = None token_type = token.type token_line = token.line token_text = token.text if token_type == lexer.ASSIGNMENT: token_attr = "ASSIGNMENT" if token_type == lexer.BEGIN: token_attr = "BEGIN" if token_type == lexer.BOOL: token_attr = "BOOL" if token_type == lexer.CLOSE_PARENTHESIS:
def getTokensFromText(input_string_to_parse, tLexer): input_stream = InputStream(input_string_to_parse) lexer = tLexer(input_stream) #lexer object is created stream = CommonTokenStream(lexer) stream.fill() return stream.tokens
def parse_model(cls, file_path=None): """ Parses a handed over model and returns the meta_model representation of it. :param file_path: the path to the file which shall be parsed. :type file_path: str :return: a new ASTNESTMLCompilationUnit object. :rtype: ASTNestMLCompilationUnit """ try: input_file = FileStream(file_path) except IOError: code, message = Messages.get_input_path_not_found(path=file_path) Logger.log_message(node=None, code=None, message=message, error_position=None, log_level=LoggingLevel.ERROR) return code, message = Messages.get_start_processing_file(file_path) Logger.log_message(node=None, code=code, message=message, error_position=None, log_level=LoggingLevel.INFO) # create a lexer and hand over the input lexer = PyNestMLLexer() lexer.removeErrorListeners() lexer.addErrorListener(ConsoleErrorListener()) lexerErrorListener = NestMLErrorListener() lexer.addErrorListener(lexerErrorListener) # lexer._errHandler = BailErrorStrategy() # N.B. uncomment this line and the next to halt immediately on lexer errors # lexer._errHandler.reset(lexer) lexer.inputStream = input_file # create a token stream stream = CommonTokenStream(lexer) stream.fill() if lexerErrorListener._error_occurred: code, message = Messages.get_lexer_error() Logger.log_message(node=None, code=None, message=message, error_position=None, log_level=LoggingLevel.ERROR) return # parse the file parser = PyNestMLParser(None) parser.removeErrorListeners() parser.addErrorListener(ConsoleErrorListener()) parserErrorListener = NestMLErrorListener() parser.addErrorListener(parserErrorListener) # parser._errHandler = BailErrorStrategy() # N.B. uncomment this line and the next to halt immediately on parse errors # parser._errHandler.reset(parser) parser.setTokenStream(stream) compilation_unit = parser.nestMLCompilationUnit() if parserErrorListener._error_occurred: code, message = Messages.get_parser_error() Logger.log_message(node=None, code=None, message=message, error_position=None, log_level=LoggingLevel.ERROR) return # create a new visitor and return the new AST ast_builder_visitor = ASTBuilderVisitor(stream.tokens) ast = ast_builder_visitor.visit(compilation_unit) # create and update the corresponding symbol tables SymbolTable.initialize_symbol_table(ast.get_source_position()) for neuron in ast.get_neuron_list(): neuron.accept(ASTSymbolTableVisitor()) SymbolTable.add_neuron_scope(neuron.get_name(), neuron.get_scope()) # store source paths for neuron in ast.get_neuron_list(): neuron.file_path = file_path ast.file_path = file_path return ast