def yield_parse_result(line1, line2): subsitution_parser = Parser('{}v{var_name} = {value};') if (result := subsitution_parser.parse(line1) ) and 'v' + result.named['var_name'] in line2 and any( op in line2 for op in STREAM_OPERATORS): var_name, value = result.named['var_name'], result.named['value'] return '', line2.replace('v' + var_name, value)
def run(): data = open_file(argv[1]) lexer = Lexer(data) tokens = lexer.lex() parser = Parser(tokens) parser.parse()
def yield_parse_result(line): std_operator_parser = Parser('{indent}std::operator{op}({params});') if (result := std_operator_parser.parse(line) ) and result.named['op'] in STREAM_OPERATORS: indent, op, params = result.named[ 'indent'], ' ' + result.named['op'], result.named['params'] return indent + op.join(params.split(',')) + ';'
def solve(self, expr: str = ""): """solve cryptarithm problem""" print("Problem: {}".format(expr)) p = Parser(tokenize(expr)) pr = Problem(p.parse()) print(pr.search_all_solution())
class Generator: output = None parser = None is_generate_code = True def __init__(self): self.parser = Parser() def set_generate_code_flag(self, is_generate_code): self.is_generate_code = is_generate_code def start(self, contracts): for contract_path in contracts: parse_info = self.parser.parse(contract_path) self.generate_documentation(parse_info) def set_output(self, output): self.output = output def gen_header(self, f, info): f.write("# " + info.contract_name + "\n") #f.write("\n`Solidity version " + info.version + "`\n") def gen_public_functions(self, f, info): f.write("\n#### Functions\n") for function in info.functions: if function.internal or function.private: continue f.write("- " + function.name + "\n") def gen_events(self, f, info): f.write("\n#### Events\n") for event in info.events: f.write("- " + event.name + "\n") #if event.notice != None: f.write("`Description:`\t" + event.notice + '\n\n') def gen_public_members(self, f, info): f.write("\n#### Members\n") for member in info.public_members: f.write("- " + member.name + " : " + member.type + "\n") #if member.notice != None: f.write("`Description:`\t" + member.notice + '\n\n') def generate_documentation(self, info): with open(self.output, 'w') as f: self.gen_header(f, info) self.gen_public_functions(f, info) self.gen_events(f, info) self.gen_public_members(f, info) f.close()
def intermediate(code): parser = Parser(code) block = parser.parse() # print(block) fd = {'params': {'var': True, 'params': []}, 'block': block} info = new_func_info(None, fd) info.add_local_var('_ENV') cg_func_def_exp(info, fd, 0) # print(info.sub_funcs[0].ins) return info
def set_parser(): """docstring for set_parser""" parse = Parser() parse.add_description('Csv plotter') parse.parser.add_argument('--csv', required=True, help='Path to the csv file') parse.parser.add_argument('--x', required=True, type=int, default=0, help='Csv file column to use for x axis') parse.parser.add_argument('--y', required=True, type=int, default=0, help='Csv file column to use for y axis') parse.parse() return parse
def run(s): scanner = Scanner(s, error) tokens = scanner.scan_tokens() parser = Parser(tokens, parse_error) statements = parser.parse() if hasError: return if hasRuntimeError: return interpreter = Interpreter() interpreter.interpret(statements)
def test_parser(self): cases = [('(1+2)<>3', ['1', '2', '+', '3', '<>']), ('1+2*3', ['1', '2', '3', '*', '+']), ('1 >= 2', ['1', '2', '>=']), ('(1*2+9) = (2*3-6)', ['1', '2', '*', '9', '+', '2', '3', '*', '6', '-', '='])] parser = Parser() for expr, correct in cases: self.assertEqual(parser.parse(expr), correct)
def preview(id): record = db_session.query(Page).get(id) if record is not None: parser = Parser() html = parser.parse(record.data) html = html.replace('</body>', '{{ post_body|safe }}</body>') last_saved = record.updated.strftime('%B %d, %Y at %I:%M%p') post_body = render_template('preview_post_body.html', last_saved=last_saved) return render_template_string(html, post_body=post_body), return render_template('empty_preview.html')
def getReply(message): p = Parser() parse_tree = p.parse(message) translation = list() find_best_translation(parse_tree, translation) print(message) print(translation) answer = " ".join(translation) # return the formulated answer return answer
class BaseRoute(Generic[_V]): """The base route class. This is referenced as `_R` in the rest of this module. # Parameters pattern (str): an URL pattern. view (_V): a view function or object whose actual type is defined by concrete routes. """ def __init__(self, pattern: str, view: _V): if pattern != WILDCARD and not pattern.startswith("/"): pattern = f"/{pattern}" self._pattern = pattern self._parser = Parser(self._pattern) self.view = view @property def pattern(self) -> str: return self._pattern def url(self, **kwargs) -> str: """Return the full URL path for the given route parameters. # Parameters kwargs (dict): route parameters. # Returns url (str): A full URL path obtained by formatting the route pattern with the provided route parameters. """ return self._pattern.format(**kwargs) def parse(self, path: str) -> Optional[dict]: """Parse an URL path against the route's URL pattern. # Returns params (dict or None): If the URL path matches the URL pattern, this is a dictionary containing the route parameters, otherwise it is `None`. """ result = self._parser.parse(path) return result.named if result is not None else None def _get_clone_kwargs(self) -> dict: return {"pattern": self._pattern, "view": self.view} def clone(self: _R, **kwargs: Any) -> _R: kwargs = {**self._get_clone_kwargs(), **kwargs} return type(self)(**kwargs)
def generate_code(path): msg = "" line_no = 0 lexer = Lexer().get_lexer() pg = Parser() pg.parse() parser = pg.get_parser() with open(path) as fp: line_no = line_no + 1 for line in fp: try: if ((len(line.replace('\n', "")) > 0) and ('#' not in line)): token = lexer.lex(line.replace('\n', "")) parser.parse(token) line_no = line_no + 1 except: msg = "Syntax error in line No " + str(line_no) return msg fp.close() Code = Codegen() write_file(Code.doit(), path) return "Successfully generated"
def make_unfiltered_rules_from_corpus(): parser = Parser() parsed_sents = set() with open("../corpus.txt", "r") as corpus_file: for line in corpus_file.readlines(): tree = parser.parse(line) parsed_sents.add(tree) ruleset = set() for iterable in parsed_sents: for tree in iterable: for rule in tree.productions(): ruleset.add(rule) return '\n'.join([str(r) for r in ruleset])
def main(args): """Main entry point for Boink. Apply necessary operations with the given arguments. Args: args (Namespace): Arguments supplied by the user. """ if args.command == 'run': try: with open(args.file, 'r') as file: # Extra newline for parsing. inp = file.read() + '\n' except FileNotFoundError: print(Colr(f"File '{args.file}' not found.", fore="red")) return # Handler for exceptions during lexing, parsing and semantic analysis. error_handler = ErrorHandler() lexer_ = Lexer(inp, error_handler) # Assign lexer to convert positions to line and offset. error_handler.lexer = lexer_ parser_ = Parser(lexer_, error_handler) # Root node of the program a.k.a. the program itself. # Argument is the program name which is equivalent to file's name. root = parser_.parse(os.path.basename(args.file)) parser_.log_tree(root) symbol_tree_builder = SymbolTableBuilder(error_handler) symbol_tree_builder.visit(root) symbol_tree_builder.log_info() if not error_handler.errors: interpreter_ = Interpreter(root, error_handler) else: error_handler.log_all() else: print(Colr(f"Command {repr(args.command)} doesn't exist", fore="red")) return
def compile(p): #Initialize symbols into a sequence of tokens lexer = Lexer(p) #Parse the regular expression parser = Parser(lexer) #Create tokens for each character in p tokens = parser.parse() #Handles NFA construction handler = Handler() nfa_stack = [] for t in tokens: handler.handlers[t.name](t, nfa_stack) #return an NFA return nfa_stack.pop()
def run(fn, text): lexer = Lexer(fn, text) tokens, error = lexer.make_tokens() if error: return None, error #Generate Abstract Syntax Tree parser = Parser(tokens) ast = parser.parse() if ast.error: return None, ast.error #Run the AST through the Interpreter interpreter = Interpreter() context = Context('<program>') context.symbol_table = global_symbol_table result = interpreter.visit(ast.node, context) return result.value, result.error
def jawbreaker(): # filter arguments arguments = parse_args() filename = arguments.filename outname = arguments.outname run = arguments.run # generate the code l = Lexer(filename) p = Parser(l) code = p.parse() # write to the file with open(outname, 'w') as out: out.write(code) # run if specified if run: os.system('python ' + outname)
def run(fn, text): lexer_ = Lexer(fn, text) tokens, error = lexer_.make_tokens() if error: return '', error print(text) print(tokens) parser = Parser(tokens) ast = parser.parse() print(ast.node) interpreter_ = Interpreter() res = interpreter_.visit(ast.node) print(res['sites'][0]['url']) return res, ''
def compile(p, debug=False): def print_tokens(tokens): for t in tokens: print(t) lexer = Lexer(p) parser = Parser(lexer) tokens = parser.parse() handler = Handler() if debug: print_tokens(tokens) nfa_stack = [] for t in tokens: handler.handlers[t.name](t, nfa_stack) assert len(nfa_stack) == 1 return nfa_stack.pop()
def main(): parser = argparse.ArgumentParser( description="SPI - Simple Pascal Interpreter") parser.add_argument("inputfile", help="Pascal source file") parser.add_argument( "--scope", help="Print scope information", action="store_true", ) parser.add_argument( "--stack", help="Print call stack", action="store_true", ) args = parser.parse_args() global _SHOULD_LOG_SCOPE, _SHOULD_LOG_STACK _SHOULD_LOG_SCOPE, _SHOULD_LOG_STACK = args.scope, args.stack text = open(args.inputfile, "r").read() # print(text) lexer = Lexer(text) try: parser = Parser(lexer) tree = parser.parse() except (LexerError, ParserError) as e: print(e.message) sys.exit(1) semantic_analyzer = SemanticAnalyzer(_SHOULD_LOG_SCOPE) try: semantic_analyzer.visit(tree) except SemanticError as e: print(e.message) sys.exit(1) interpreter = Interpreter() interpreter.interpret(tree)
def compile(p, debug = False): def print_tokens(tokens): for t in tokens: print(t) lexer = Lexer(p) parser = Parser(lexer) tokens = parser.parse() handler = Handler() if debug: print_tokens(tokens) nfa_stack = [] for t in tokens: handler.handlers[t.name](t, nfa_stack) assert len(nfa_stack) == 1 return nfa_stack.pop()
def publish(id): record = db_session.query(Page).get(id) if record is not None: parser = Parser() html = parser.parse(record.data) if record.slug is not None: dir_name = record.slug else: dir_name = record.key if record.access == 'public': file_path = '%s%s' % (config['generate']['public_path'], dir_name) url = dir_name elif record.access == 'limited': file_path = '%s%s' % (config['generate']['limited_path'], dir_name) url = '%s?key=%s' % (dir_name, record.key) else: # access is private so do nothing return jsonify(published=record.published.strftime('%B %d, %Y at %I:%M%p')) if not os.path.exists(file_path): os.makedirs(file_path) full_path = '%s/index.html' % file_path with open(full_path, "w") as fh: fh.write(html) # set published date record.published = datetime.now() db_session.add(record) db_session.commit() return jsonify( published_date=record.published.strftime('%B %d, %Y at %I:%M%p'), url=url ) return jsonify(error=True, message="Not found")
async def solve_cryptarithm(expr: str = '') -> Dict[str, Any]: p = Parser(tokenize(expr)) pr = Problem(p.parse()) status, solutions = pr.search_all_solution() return {"problem": expr, "answer": solutions, "status": status}
tile_height = 6 row = 1 col = 8 left_start = origin + 0 * tile_width # top_start = origin + 6 * tile_height top_start = origin + 70 + 4 scenes = [] tasks = [] for left in range(left_start, left_start + col * tile_width, tile_width): for top in range(top_start, top_start + row * tile_height, tile_height): parser = Parser() s = parser.parse(testfile) scenes.append(s) scenes[-1].sampler = TileSampler(left, top, tile_width, tile_height) name = out_dir + '%d.%d.png' % (left, top) scenes[-1].film = copy.deepcopy(scenes[-1].film) scenes[-1].film.filename = name tasks.append(scenes[-1].render) p = Pool(len(tasks)) for s in scenes: print(s.film.filename) print('%d, %d' % (s.sampler.left, s.sampler.top)) for task in tasks:
use_stemming = False for o, a in opts: if o in ("-s", "--stemming"): use_stemming = True print "use stemming: %d" % use_stemming filelist = [(path + f) for f in os.listdir(path)] parser = Parser(fstopname) for stem in [use_stemming]: for idf in True, False: print "Parsing files...", stdout.flush() parser.parse(filelist, stem) # Ignore the 10% least and most frequent words parser.words = slice_sorted_words(parser.words, 30) print "done" print "Normalizing frequencies...", stdout.flush() # Don't modify the original set for i, doc in enumerate(parser.docset): normalize(doc, parser.words, idf) print i gc.collect() print "done" for chooser in choose_initial_pp, choose_initial: for k in 10, 20, 30, 40:
dot.edge(p_name, name) for n in node.params: see_node(name, n, dot) see_node(name, node.body, dot) for n in node.localvars: see_node(name, n, dot) elif type(node) == CompoundStmtNode: dot.node(name, str(node.kind)) dot.edge(p_name, name) for n in node.stmts: see_node(name, n, dot) elif type(node) == IfStmtNode: dot.node(name, str(node.kind)) dot.edge(p_name, name) see_node(name, node.cond, dot) see_node(name, node.then, dot) if not node.els is None: see_node(name, node.els, dot) if __name__ == '__main__': path = './t1.c' l = Lexer(filename(path), read_file(path)) l.lex() l.see_tokens() p = Parser(l.tokens) p.parse() see_ast(p.ast)
from pascal_loader.main_io import PascalFile from parse import Parser from emulator import Emulator if __name__ == '__main__': pretty_printer = pprint.PrettyPrinter() # UNCOMMENT the below statements one at a time # tokens = get_token(PascalFile(input_file_location='simple_assignment.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='complex_assignments.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='control_repeat.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='control_while.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='control_if.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='control_for.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='case_statement.pas', output_location='')) tokens = get_token(PascalFile(input_file_location='arrays.pas', output_location='')) # This prints tokens, uncomment to see the generated tokens # pretty_printer.pprint(tokens) print '----------------------------------' # setting verbose=True to parser will print to console as tokens are matched/warnings # parser = Parser(token_list=tokens, verbose=True) parser = Parser(token_list=tokens) byte_array = parser.parse() # This prints the byte array, uncomment to see the bytearray # pretty_printer.pprint(byte_array) print '----------------------------------' emulator = Emulator(byte_array) emulator.start()
precedence = { '||': 5, '&&': 5, '>': 6, '>=': 6, '<=': 6, '<': 6, '==': 6, '!=': 6, '+': 7, '-': 7, '*': 8, '/': 8, 'UMINUS': 9 } tokens = list(Lexer('1+-10*123').lex()) precs = {'UMINUS': ['E', '-', 'E'], 'POSITIVE': ['E', '+', 'E']} # for t in tokens: # print(t) # print(sm.productions()) # parser = Parser(productions, terminal, nonterminal) parser = Parser(sm.productions, sm.terminal, sm.nonterminal, precedence=precedence, precs=precs) parser.generate() parser.parse(tokens, sm.sdmap) print(calls)
from parse import Parser parser = Parser(file="cfg") parser.parse()
from lexer import Lexer from parse import Parser from codegen import CodeGen text_input = """ print(4 + 4 - 2); """ lexer = Lexer().get_lexer() tokens = lexer.lex(text_input) codegen = CodeGen() module = codegen.module builder = codegen.builder printf = codegen.printf pg = Parser(module, builder, printf) pg.parse() parser = pg.get_parser() parser.parse(tokens).eval() codegen.create_ir() codegen.save_ir("output.ll")
# print(sm.terminal) t2p = {'id': '[a-zA-Z_]\w*', 'num': '\d+'} # lexer = Lexer('node/test3.dm',sm.terminal,t2p) lexer = Lexer('node/test6.dm', sm.terminal, t2p) # lexer = Lexer('test2.dm',sm.terminal,t2p) # print(list(lexer.lex())) # for t in lexer.lex(): # print(t) # parser.generate() # parser.dumpjson() parser.loadjson() # parser.htmlparse('test.html') tokens = list(lexer.lex()) tree = parser.parse(tokens, sm.sdmap) #, debug=True) # typeCheck = TypeCheck(tree) # typeCheck.init() # typeCheck.accept() # inter = Interperter(tree) # inter.accept() # Parser.printitems(sm.productions) # cnt = Counter([p[0] for p in sm.productions]) # print(cnt) past = AstPrintVisitor(tree) past.accept() # gast = AstTraversalVisitor(tree) # print(list(gast.accept())) print(calls)
use_stemming = False for o, a in opts: if o in ('-s','--stemming'): use_stemming = True print 'use stemming: %d' % use_stemming filelist = [(path + f) for f in os.listdir(path)] parser = Parser(fstopname) for stem in [use_stemming,]: for idf in True, False: print 'Parsing files...', stdout.flush() parser.parse(filelist[:10000], stem) # Ignore the 30% least and most frequent words parser.words = slice_sorted_words(parser.words, 30) print 'done' print 'Normalizing frequencies...', stdout.flush() # Don't modify the original set for i, doc in enumerate(parser.docset): normalize(doc, parser.words, idf) print i gc.collect() print 'done' for chooser in [choose_initial]: #choose_initial_pp, choose_initial: for k in [10]:
from Pascal_Helper_Files.pascal_reader import PascalFile from parse import Parser # parser from emulator import Emulator # emulator if __name__ == '__main__': pretty_printer = pprint.PrettyPrinter() # UNCOMMENT the below statements one at a time # tokens = get_token(PascalFile(input_file_location='array_example.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='assignment_example.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='for_example.pas', output_location='')) # tokens = get_token(PascalFile(input_file_location='if_example.pas', output_location='')) tokens = get_token( PascalFile(input_file_location='while_example.pas', output_location='')) # UNCOMMENT THE LINE BELOW TO TEST THE SCANNER --> YOU WILL SEE THE TOKENS # pretty_printer.pprint(tokens) print '----------------------------------' # UNCOMMENT THE LINE BELOW TO TEST THE PARSER # parser = Parser(token_list=tokens, verbose=True) parser = Parser(token_list=tokens) byte_array = parser.parse() # This prints the byte array, uncomment to see the bytearray # pretty_printer.pprint(byte_array) print '----------------------------------' emulator = Emulator(byte_array) emulator.start()
# starts at root of the of the parse tree and checks if the # phrase at each node is in the idiom cache or dictionary def find_best_translation(parse_tree, translation): for root in parse_tree: if(isinstance(root, str)): translation.append(root) return phrase = " ".join(root.leaves()) class_of = find_class_of(phrase) if find_in_cache(phrase) != None: translation.append(__idiom_dict[phrase]) elif class_of != None: translation.append(find_target_language_translation(class_of, "mandarin")) else: find_best_translation(root, translation) return if __name__ == '__main__': p = Parser() parse_tree = p.parse("I am cleaning the bathroom") translation = list() find_best_translation(parse_tree, translation) print(translation)
from lex import Lexer from parse import Parser from codegen import CodeGen fname = "input.toy" with open(fname) as f: text_input = f.read() lexer = Lexer().get_lexer() tokens = lexer.lex(text_input) codegen = CodeGen() module = codegen.module builder = codegen.builder printf = codegen.printf #~ pg = Parser(module, builder, printf) pg1 = Parser() pg1.parse() parser = pg1.get_parser() parser.parse(tokens) codegen.create_ir() #~ codegen.save_ir("output.ll")