def convert(inputs: List[Input]) -> str: result = "" if len(inputs) <= 1 else Generator("bootstrap").bootstrap() for input in inputs: generator = Generator(input.namespace) visitor = Visitor(generator) if result is None: result = generator.bootstrap() lexer = VMLexer(input.stream) tokens = antlr4.CommonTokenStream(lexer) parser = VMParser(tokens) tree = parser.program() result = result + visitor.visit(tree) return result
def from_sstream(sstream: str, translator: nars2networkx) -> None: ## Create lexer and tokenize input # print("Create lexer for...") # print(sstream) lexer = NarseseLexer(antlr.InputStream(sstream)) # print("Tokenize...") token_stream = antlr.CommonTokenStream(lexer) ## Create parser and build AST # print("Create parser...") parser = NarseseParser(token_stream) parser._interp.predictionMode = antlr.PredictionMode.SLL # print("Parse...") tree = parser.narsese() ## Create Tree walker and inject the translator over a networkx graph walker = antlr.ParseTreeWalker() ## Roll # print("Walk tree...") walker.walk(translator, tree)
def process(self, infile, outfile): """ Processes a SDoc1 document. :param str infile: The input filename with the SDoc2 document. :param str outfile: The output filename with the target document. """ in_stream = antlr4.FileStream(infile, 'utf-8') lexer = sdoc2Lexer(in_stream) tokens = antlr4.CommonTokenStream(lexer) parser = sdoc2Parser(tokens) tree = parser.sdoc() visitor = SDoc2Visitor() visitor.visit(tree) sdoc.sdoc2.node_store.prepare_content_tree()
def ParseFile(self): self.PreprocessFile() # restore from ListOfList to ListOfString self.Profile.FileLinesList = [ "".join(list) for list in self.Profile.FileLinesList ] FileStringContents = '' for fileLine in self.Profile.FileLinesList: FileStringContents += fileLine for Token in self.TokenReleaceList: if Token in FileStringContents: FileStringContents = FileStringContents.replace( Token, 'TOKENSTRING') cStream = antlr.InputStream(FileStringContents) lexer = CLexer(cStream) tStream = antlr.CommonTokenStream(lexer) parser = CParser(tStream) parser.translation_unit()
def parse(input_stream): lexer = AgentScriptCCLexer(input_stream) stream = antlr4.CommonTokenStream(lexer) parser = AgentScriptCCParser(stream) parser.removeErrorListeners() errorListener = AgentScriptCCErrorListener() parser.addErrorListener(errorListener) tree = parser.program() loader = AgentScriptCCLoader() walker = antlr4.ParseTreeWalker() walker.walk(loader, tree) program = loader.program program.parsing_errors = errorListener.errors return program
def parse(self): """ Parse the input query and store the output in self.tree. """ inpt = antlr4.InputStream(self.query) lexer = ADQLLexer(inpt) self.stream = antlr4.CommonTokenStream(lexer) self.parser = ADQLParser(self.stream) self.syntax_error_listener = SyntaxErrorListener() self.parser._listeners = [self.syntax_error_listener] self.tree = self.parser.query() if len(self.syntax_error_listener.syntax_errors): raise QuerySyntaxError(self.syntax_error_listener.syntax_errors) self.walker = antlr4.ParseTreeWalker()
def compile(filename): # prepare file to be parsed inputfile = a4.FileStream(filename) lexer = ClassLayoutLexer(inputfile) stream = a4.CommonTokenStream(lexer) # parse to a tree parser = ClassLayoutParser(stream) tree = parser.u2cFile() # open the listener fl = FileListener() # walk the tree walker = a4.ParseTreeWalker() walker.walk(fl, tree) print(fl.c.write('java'))
def main(): # Setup lexers and parsers lexer = FrackLexer(antlr4.FileStream(TEST_FILE)) stream = antlr4.CommonTokenStream(lexer) parser = FrackParser(stream) tree = parser.program() # Actually parse the program listener = Listener() walker = antlr4.ParseTreeWalker() #walker.walk(listener, tree) print(tree.toStringTree()) for f in listener.functions: print(f) print('done!')
def compile(nndl_content): """ """ input_stream = antlr4.InputStream(nndl_content) lexer = NNDLLexer.NNDLLexer(input_stream) stream = antlr4.CommonTokenStream(lexer) parser = NNDLParser.NNDLParser(stream) tree = parser.prog() # Walk the tree and generate the dot file dg = DotGenerator() walker = antlr4.ParseTreeWalker() walker.walk(dg, tree) # Use the dotgenerator's network that it figured out from the nndl file # to generate the cpp file nw = dg._network return nw
def process(self, infile: str) -> int: """ Processes a SDoc1 document and returns the error count. :param str infile: The input filename with the SDoc2 document. """ in_stream = antlr4.FileStream(infile, 'utf-8') lexer = sdoc2Lexer(in_stream) tokens = antlr4.CommonTokenStream(lexer) parser = sdoc2Parser(tokens) tree = parser.sdoc() visitor = SDoc2Visitor(infile, self._io) visitor.visit(tree) sdoc2.node_store.prepare_content_tree() return visitor.errors
def main(argv): if len(argv) != 3: raise AttributeError('invalid number of arguments to compiler') input_file, project_dir = argv[1:] if not input_file.endswith('.lat'): raise AttributeError('input_file must have `.lat` extension') out_path = os.path.dirname(input_file) base_name = os.path.split(input_file)[1][:-4] out_base_name = os.path.join(out_path, base_name) input_file_stream = antlr4.FileStream(input_file) syntax_error_listener = LatteParserErrorListener() lexer = LatteLexer(input_file_stream) lexer.removeErrorListeners() lexer.addErrorListener(syntax_error_listener) token_stream = antlr4.CommonTokenStream(lexer) parser = LatteParser(token_stream) parser.removeErrorListeners() parser.addErrorListener(syntax_error_listener) prog_tree = parser.program() compiler = LLVMCompiler() code = compiler.visit_prog(prog_tree) print('OK', file=sys.stderr) ll_file_path = out_base_name + '.ll' runtime_path = os.path.join(project_dir, 'lib', 'runtime.bc') bc_no_runtime_path = out_base_name + '_no_runtime.bc' bc_final_path = out_base_name + '.bc' with open(ll_file_path, 'w') as f: f.write(code) print(f'Saved {ll_file_path}') if os.system(f'llvm-as -o {bc_no_runtime_path} {ll_file_path}') != 0: sys.exit(3) print(f'Compiled to {bc_no_runtime_path}') if os.system(f'llvm-link -o {bc_final_path} ' f'{bc_no_runtime_path} {runtime_path}') != 0: sys.exit(4) os.remove(bc_no_runtime_path) print(f'Linked to runtime: {bc_final_path}')
def __call__(self, parser, namespace, values, option_string=None): from azure.cli.command_modules.monitor.grammar import ( MetricAlertConditionLexer, MetricAlertConditionParser, MetricAlertConditionValidator) string_val = ' '.join(values) lexer = MetricAlertConditionLexer(antlr4.InputStream(string_val)) stream = antlr4.CommonTokenStream(lexer) parser = MetricAlertConditionParser(stream) tree = parser.expression() validator = MetricAlertConditionValidator() walker = antlr4.ParseTreeWalker() walker.walk(validator, tree) metric_condition = validator.result() super(MetricAlertConditionAction, self).__call__(parser, namespace, metric_condition, option_string)
def create_program(self, templatefilepath, directory, filename): tmpfilepath = '/tmp/' + templatefilepath.split('/')[-1] with open(tmpfilepath, 'w') as tmpfile: tmpfile.write(ReOrder(templatefilepath).reordered_code) template = antlr4.FileStream(tmpfilepath) lexer = Template2Lexer(template) stream = antlr4.CommonTokenStream(lexer) parser = Template2Parser(stream) template = parser.template() walker = ParseTreeWalker() walker.walk(self, template) with open( directory + '/' + filename.split('/')[-1].replace('.template', '.py'), 'w') as modelfile: modelfile.write(self.output_program)
def process(self, infile, outfile): """ Processes a SDoc1 document. :param str infile: The input filename with the SDoc1 document. :param str outfile: The output filename with the SDoc2 document. """ in_stream = antlr4.FileStream(infile) out_stream = open(outfile, 'wt') lexer = sdoc1Lexer(in_stream) tokens = antlr4.CommonTokenStream(lexer) parser = sdoc1Parser(tokens) tree = parser.sdoc() visitor = SDoc1Visitor( root_dir=os.path.dirname(os.path.realpath(infile))) visitor.set_output(out_stream) visitor.visit(tree)
def process_sympy(sympy): matherror = MathErrorListener(sympy) stream = antlr4.InputStream(sympy) lex = PSLexer(stream) lex.removeErrorListeners() lex.addErrorListener(matherror) tokens = antlr4.CommonTokenStream(lex) parser = PSParser(tokens) # remove default console error listener parser.removeErrorListeners() parser.addErrorListener(matherror) relation = parser.math().relation() expr = convert_relation(relation) return expr
def main(): test1 = ''' &foo; ; & fooo; 5 bytes; 10 bytes be; &foo bits; 10;''' test2 = '!(5 < 6) || 4' test3 = '[ sint : 5 bytes be ]; [uint : 5 bytes: &foo];' test4 = '[sint::5];' test5 = ''' packet P2_generic : P2(&subtype == 3) { always timestamp [ uint: 4 bytes be ]; always version [ uint: 2 bits ]; always _padding [ _ : 6 bits]; // pad to align to next byte variable speed { case(&version < 1) | v1: 0 | { always speed [ uint : 2 bytes be ]; } case(&version < 2) | v2: 1 | { always speed [ uint : 3 bytes be ]; } otherwise | cur: 2 | { always speed [ uint : 4 bytes be ]; } } optional metadata when(&version > 3) { always distance [ uint : 4 bytes be ]; always end_time [ uint : 4 bytes be ]; } }''' inp = antlr4.InputStream(test5) lexer = TightLexer(inp) stream = antlr4.CommonTokenStream(lexer) parser = TightParser(stream) tree = parser.module() print(tree.toStringTree(TightParser.ruleNames))
def get_pyro_features(self, pyrofile_path): import antlr4 from antlr4 import * from parser.Python3Parser import Python3Parser from parser.Python3Lexer import Python3Lexer pyrofile = antlr4.FileStream(pyrofile_path) lexer = Python3Lexer(pyrofile) stream = antlr4.CommonTokenStream(lexer) parser = Python3Parser(stream) code = parser.file_input() walker = ParseTreeWalker() walker.walk(self, code) feature_vector = {} for k in self.post_map: if 'post' in self.post_map[k]: feature_vector['vi_' + self.post_map[k]['prior'] + '_' + self.post_map[k]['post']] = 1 return feature_vector
def main(): # command line parser = argparse.ArgumentParser(description='Exec/Type mu files.') parser.add_argument('path', type=str, help='file to exec and type') args = parser.parse_args() # lex and parse input_s = antlr4.FileStream(args.path, encoding='utf8') lexer = MuLexer(input_s) stream = antlr4.CommonTokenStream(lexer) parser = MuParser(stream) tree = parser.prog() # eval Visitor - You have some TODOS in this file! visitor2 = MuEvalVisitor() try: visitor2.visit(tree) except (MuRuntimeError, MuSyntaxError) as e: print(e.args[0]) exit(1)
def prg2py_after_preproc(data, parser_start, input_filename): input_stream = antlr4.InputStream(data) lexer = VisualFoxpro9Lexer(input_stream) stream = antlr4.CommonTokenStream(lexer) parser = VisualFoxpro9Parser(stream) tree = run_parser(stream, parser, parser_start) TreeCleanVisitor().visit(tree) output_tree = PythonConvertVisitor(input_filename).visit(tree) if not isinstance(output_tree, list): return output_tree output = add_indents(output_tree, 0) options = autopep8.parse_args(['--max-line-length', '100000', '-']) output = autopep8.fix_code(output, options) tokens = list(tokenize.generate_tokens(io.StringIO(output).readline)) for i, token in enumerate(tokens): token = list(token) if token[0] == tokenize.STRING and token[1].startswith('u'): token[1] = token[1][1:] tokens[i] = tuple(token) return tokenize.untokenize(tokens)
def initialize_transform_rules(): # 文件中的自定义转换规则 rule_file = './rule.txt' input = antlr4.FileStream(rule_file) lexer = rulesLexer(input) stream = antlr4.CommonTokenStream(lexer) parser = rulesParser(stream) tree = parser.rules() rules = set() for single_rule in tree.getChildren(): # 这里的child下标和g4文件定义的规则相关 rule_name = single_rule.getChild(0).getText() # 最前面的部分是规则名称 origin_parse_tree = expr.ParseTree( single_rule.getChild(2)) # input pattern transformed_parse_tree = expr.ParseTree( single_rule.getChild(4)) # output pattern if single_rule.getChildCount() == 10: cp_parse_tree = None pp_parse_tree = None weight = int(single_rule.getChild(8).getText()[1:]) elif single_rule.getChildCount() == 11: if single_rule.getChild(6).getText() != '@': cp_parse_tree = expr.ParseTree(single_rule.getChild(6)) pp_parse_tree = None else: cp_parse_tree = None pp_parse_tree = expr.ParseTree(single_rule.getChild(7)) weight = int(single_rule.getChild(9).getText()[1:]) else: cp_parse_tree = expr.ParseTree(single_rule.getChild(6)) pp_parse_tree = expr.ParseTree(single_rule.getChild(8)) weight = int(single_rule.getChild(10).getText()[1:]) rules.add( TransformRule(rule_name, origin_parse_tree, transformed_parse_tree, cp_parse_tree, pp_parse_tree, weight)) return rules
def _parse_input_stream(input_stream:antlr4.InputStream) -> RootNode: error_listener = _ConsoleErrorListener() lexer = JSONPathLexer(input_stream) lexer.addErrorListener(error_listener) token_stream = antlr4.CommonTokenStream(lexer) parser = _JSONPathParser(token_stream) parser.addErrorListener(error_listener) tree = parser.jsonpath() listener = _JSONPathListener(_stack=[]) walker = antlr4.ParseTreeWalker() walker.walk(listener, tree) return listener._stack.pop()
def main(argv): if len(argv) != 4: raise AttributeError('invalid number of arguments to compiler') input_file, target_vm, project_dir = argv[1:] if not input_file.endswith('.ins'): raise AttributeError('input_file must have `ins` extension') out_path = os.path.dirname(input_file) base_name = os.path.split(input_file)[1][:-4] out_base_name = os.path.join(out_path, base_name) input_file_stream = antlr4.FileStream(input_file) lexer = InstantLexer(input_file_stream) token_stream = antlr4.CommonTokenStream(lexer) parser = InstantParser(token_stream) prog_tree = parser.prog() if target_vm == 'jvm': compiler = JVMCompiler(base_name) elif target_vm == 'llvm': compiler = LLVMCompiler() else: raise AttributeError(f'unknown target VM: `{target_vm}`') code = compiler.visit_prog(prog_tree) if target_vm == 'llvm': ll_file_path = out_base_name + '.ll' bc_file_path = out_base_name + '.bc' with open(ll_file_path, 'w') as f: f.write(code) print(f'Saved {ll_file_path}') os.system(f'llvm-as {ll_file_path} -o {bc_file_path}') print(f'Compiled to {bc_file_path}') elif target_vm == 'jvm': j_file_path = out_base_name + '.j' with open(j_file_path, 'w') as f: f.write(code) print(f'Saved {j_file_path}') jasmin_path = os.path.join(project_dir, 'lib', 'jasmin.jar') os.system(f'java -jar {jasmin_path} -d {out_path} {j_file_path}')
def visitCmd_include(self, ctx): """ Includes another SDoc into this SDoc. :param sdoc1Parser.Cmd_includeContext ctx: The parse tree. """ # Test the maximum include level. if self._include_level >= self._options['max_include_level']: raise RuntimeError( "Maximum include level exceeded." ) # @todo More verbose logging, own exception class. # Open a stream for the sub-document. file_name = sdoc.unescape(ctx.SIMPLE_ARG().getText()) if not os.path.isabs(file_name): file_name = os.path.join(self._root_dir, file_name + '.sdoc') print("Including %s" % os.path.relpath(file_name)) stream = antlr4.FileStream(file_name, 'utf-8') # root_dir # Create a new lexer and parser for the sub-document. lexer = sdoc1Lexer(stream) tokens = antlr4.CommonTokenStream(lexer) parser = sdoc1Parser(tokens) tree = parser.sdoc() # Create a visitor. visitor = SDoc1Visitor( root_dir=os.path.dirname(os.path.realpath(file_name))) # Set or inherit properties from the parser of the parent document. visitor._include_level = self._include_level + 1 visitor.set_output(self._output) visitor._set_global_scope(self._global_scope) # Run the visitor on the parse tree. visitor.visit(tree) self.put_position(ctx, 'stop')
def main(argv): input_filename = argv[1] output_filename = argv[2] input_file = AntlrCaseInsensitiveFileInputStream(input_filename) lexer = PlSqlLexer(input_file) stream = antlr4.CommonTokenStream(lexer) parser = PlSqlParser(stream) tree = parser.sql_script() visitor = ScriptVisitor() node = tree.accept(visitor) #print(ast.dump(node)) #astpretty.pprint(node) # este modulo esta malo. no usar :( try: code = astor.to_source(node) except: print(ast.dump(node)) raise output = open(output_filename, "w") output.write(code) output.close()
def casos_de_teste_semantico(): with open('oi.txt', 'r') as caso_de_teste: mapa = caso_de_teste.read() mapa_input = antlr4.InputStream(mapa) lexer = TileMapLexer(input=mapa_input) lexer.removeErrorListeners() tokens = antlr4.CommonTokenStream(lexer=lexer) parser = TileMapParser(tokens) parser.removeErrorListeners() erros_sintaticos = ErrosSintaticosErrorListener() parser.addErrorListener(erros_sintaticos) try: mapa = parser.mapa() analisador_semantico = Semantico() analisador_semantico.visitMap(mapa) print('Compilação finalizada') except Exception as e: print(str(e), file=sys.stderr) pass
def _parse_rule(self): stream = antlr4.InputStream(self.rule) fl = CFLexer(stream) fl.removeErrorListeners() fl.addErrorListener(FilterErrorListener.INSTANCE) token_stream = antlr4.CommonTokenStream(fl) tree = CFilter(token_stream) tree.removeErrorListeners() tree.addErrorListener(FilterErrorListener.INSTANCE) visitor = MyVisitor() self.identifiers, self.node = visitor.visit_res(tree.root()) # identifiers is None or len(self.identifiers) == 0 # 表示表达式没有变量,所以可以直接计算出表达式的值 # 此时可以提前计算表达式的值的函数更换visit_res为visit_const_res if self.identifiers is None or len(self.identifiers) == 0: self.node = self.node.visit() self.__dict__["visit_res"] = lambda *args, **kwargs: self.node self.is_const = True else: self.is_const = False
def casos_de_teste_gerador(): print('-------------------------------------') print('CASOS DE TESTE DO GERADOR DE CÓDIGO ') print('------------------------------------') for i in range(1, 2): with open(DIRETORIO_PROJETO + CAMINHO_ARQUIVOS_ENTRADA + GERACAO_DE_CODIGO + 'ct_gerador_' + str(i) + '.txt', encoding='utf-8') as caso_de_teste: programa = caso_de_teste.read() programa_input = antlr4.InputStream(programa) lexer = tileLexer(input=programa_input) lexer.removeErrorListeners() tokens = antlr4.CommonTokenStream(lexer=lexer) parser = tileParser(tokens) parser.removeErrorListeners() erros_sintaticos = ErrosSintaticosErrorListener() parser.addErrorListener(erros_sintaticos) try: programa = parser.mapa() gerador_de_codigo = GeradorCodigo() gerador_de_codigo.visitMapa(programa) codigo_gerado = gerador_de_codigo.getCodigo() arquivo = DIRETORIO_PROJETO + CAMINHO_ARQUIVOS_SAIDA + GERACAO_DE_CODIGO + 'ct_gerador_' + \ str(i) + '.html' arquivo_saida = open(arquivo, 'w', encoding='utf-8') arquivo_saida.write(codigo_gerado) arquivo_saida.close() print('[CT' + str(i) + '_GERADOR] Codigo gerado!') except Exception as e: print('[CT' + str(i) + '_GERADOR] ' + str(e), file=sys.stderr) pass
def parse_ctc(self, ctc: str): tree = None words = ctc.split(' ') final_ctc = '' aux = False for word in words: if word == 'NOT' or word == '(NOT': final_ctc = final_ctc + ' (' + word + ' (' aux = True elif aux: final_ctc = final_ctc + word + ')) ' aux = False else: final_ctc = final_ctc + " " + word final_ctc = final_ctc[1:] lexer = propositionalLexer(antlr4.InputStream(final_ctc)) stream = antlr4.CommonTokenStream(lexer) parser = propositionalParser(stream) tree = parser.formula() return tree
def create_program(self): template = antlr4.FileStream(self.templatefile) lexer = TemplateLexer(template) stream = antlr4.CommonTokenStream(lexer) parser = TemplateParser(stream) self.visit(parser.template()) # write data with open(self._directory + '/data.json', 'w') as datafile: for k in six.iterkeys(self.data_json): if isinstance(self.data_json[k], np.ndarray): self.data_json[k] = self.data_json[k].tolist() json.dump(self.data_json, datafile) with open(self._directory + '/model.stan', 'w') as modelfile: modelfile.write(self.output_program) modelfile.write(self.model_string) try: if self.config["stan"]["quants"] is True: modelfile.write(self.add_quants()) except: pass
def parse_query(query_str): """ Parses GAE search query and returns easily readable composition of Expressions. Args: query_str: a str representing GAE search query. Returns: An Expression or ExpressionsGroup corresponding to query_str. """ input_stream = antlr4.InputStream(query_str) lexer = queryLexer(input_stream) stream = antlr4.CommonTokenStream(lexer) parser = queryParser(stream) raw_query_node = parser.query() # MATCHED: exprs_seq EOF if parser.getNumberOfSyntaxErrors(): if 'distance(' in query_str: msg = 'Searching on GeoPoint fields is not supported yet.' else: msg = 'Failed to parse query string: "{}"'.format(query_str) raise InvalidRequest(msg) exprs_seq_node = raw_query_node.children[0] return _process_exprs_seq(exprs_seq_node)