def PrintAst(node, flag): # type: (command_t, arg_types.main) -> None if flag.ast_format == 'none': stderr_line('AST not printed.') if 0: from _devbuild.gen.id_kind_asdl import Id_str from frontend.lexer import ID_HIST for id_, count in ID_HIST.most_common(10): print('%8d %s' % (count, Id_str(id_))) print() total = sum(ID_HIST.values()) print('%8d total tokens returned' % total) else: # text output f = mylib.Stdout() afmt = flag.ast_format # note: mycpp rewrite to avoid 'in' if afmt in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif afmt in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError() if 'abbrev-' in afmt: tree = node.AbbreviatedTree() else: tree = node.PrettyTree() ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n')
def t_parse( s, # type: str expected=None, # type: Optional[str] ): # type: (...) -> arith_expr_t p = typed_arith_parse.MakeParser(s) tree = p.Parse() print() print('---') print() print(s) print() print(tree) import sys from asdl import format as fmt ast_f = fmt.DetectConsoleOutput(sys.stdout) p_node = tree.AbbreviatedTree() fmt.PrintTree(p_node, ast_f) #v = PrettyPrinter() #v.Visit(tree) #print('%-40s %s' % (s, sexpr)) return tree
def main(argv): # type: (List[str]) -> int arena = alloc.Arena() arena.PushSource(source.Stdin('')) parse_opts = parse_lib.OilParseOptions() # Dummy value; not respecting aliases! aliases = {} # type: Dict[str, Any] # parse `` and a[x+1]=bar differently loader = pyutil.GetResourceLoader() oil_grammar = meta.LoadOilGrammar(loader) parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar, one_pass_parse=True) line_reader = reader.FileLineReader(mylib.Stdin(), arena) c_parser = parse_ctx.MakeOshParser(line_reader) try: node = main_loop.ParseWholeFile(c_parser) except util.ParseError as e: ui.PrettyPrintError(e, arena) return 2 assert node is not None tree = node.AbbreviatedTree() #tree = node.PrettyTree() ast_f = fmt.DetectConsoleOutput(mylib.Stdout()) fmt.PrintTree(tree, ast_f) ast_f.write('\n') return 0
def main(argv): # type: (List[str]) -> int pool = alloc.Pool() arena = pool.NewArena() arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) # Dummy value; not respecting aliases! aliases = {} # type: Dict[str, Any] # parse `` and a[x+1]=bar differently parse_ctx = parse_lib.ParseContext(arena, aliases, one_pass_parse=True) c_parser = parse_ctx.MakeOshParser(line_reader) try: node = main_loop.ParseWholeFile(c_parser) except util.ParseError as e: ui.PrettyPrintError(e, arena) return 2 assert node is not None tree = node.AbbreviatedTree() #tree = node.PrettyTree() ast_f = fmt.DetectConsoleOutput(sys.stdout) fmt.PrintTree(tree, ast_f) ast_f.write('\n') return 0
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') if action == 'py': # Prints the module # Called by asdl/run.sh py-cpp schema_path = argv[2] app_types = {'id': asdl.UserType('id_kind_asdl', 'Id_t')} with open(schema_path) as f: schema_ast, type_lookup = front_end.LoadSchema(f, app_types) root = sys.modules[__name__] # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a # demo. py_meta.MakeTypes(schema_ast, root, type_lookup) log('AST for this ASDL schema:') schema_ast.Print(sys.stdout, 0) print() log('Dynamically created a Python module with these types:') for name in dir(root): print('\t' + name) if 1: # NOTE: It can be pickled, but not marshaled import marshal import cPickle print(dir(marshal)) out_path = schema_path + '.pickle' with open(out_path, 'w') as f: #marshal.dump(type_lookup, f) # Version 2 is the highest protocol for Python 2.7. cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2) print('runtime_type_lookup:') for name, desc in type_lookup.runtime_type_lookup.items(): print(name) print(desc) print() print('Wrote %s' % out_path) elif action == 'arith-format': # pretty printing expr = argv[2] obj = typed_arith_parse.ParseShell(expr) tree = obj.PrettyTree() #treee= ['hi', 'there', ['a', 'b'], 'c'] f = fmt.DetectConsoleOutput(sys.stdout) fmt.PrintTree(tree, f) print() # Might need to print the output? # out.WriteToFile? else: raise RuntimeError('Invalid action %r' % action)
def PrintAst(nodes, opts): if len(nodes) == 1: node = nodes[0] else: node = command.CommandList(nodes) if opts.ast_format == 'none': print('AST not printed.', file=sys.stderr) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError abbrev_hook = (ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None) tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook) ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n')
def __repr__(self): ast_f = fmt.TextOutput(util.Buffer()) # No color by default. #ast_f = fmt.AnsiOutput(io.StringIO()) tree = fmt.MakeTree(self) fmt.PrintTree(tree, ast_f) s, _ = ast_f.GetRaw() return s
def PrintAst(nodes, opts): if len(nodes) == 1: node = nodes[0] else: node = ast.CommandList(nodes) if opts.ast_format == 'none': print('AST not printed.', file=sys.stderr) elif opts.ast_format == 'oheap': # TODO: Make this a separate flag? if sys.stdout.isatty(): raise RuntimeError('ERROR: Not dumping binary data to a TTY.') f = sys.stdout enc = encode.Params() out = encode.BinOutput(f) encode.EncodeRoot(node, enc, out) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError abbrev_hook = (ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None) tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook) ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n')
def main(argv): # type: (List[str]) -> int arena = alloc.Arena() arena.PushSource(source.Stdin('')) loader = pyutil.GetResourceLoader() oil_grammar = meta.LoadOilGrammar(loader) parse_ctx = None e_parser = expr_parse.ExprParser(parse_ctx, oil_grammar) line_lexer = lexer.LineLexer('', arena) line_reader = reader.FileLineReader(sys.stdin, arena) lex = lexer.Lexer(line_lexer, line_reader) try: pnode, _ = e_parser.Parse(lex, grammar_nt.command_expr) except error.Parse as e: ui.PrettyPrintError(e, arena) return 2 #print(pnode) tr = expr_to_ast.Transformer(oil_grammar) node = tr.Expr(pnode) assert node is not None tree = node.AbbreviatedTree() #tree = node.PrettyTree() ast_f = fmt.DetectConsoleOutput(sys.stdout) fmt.PrintTree(tree, ast_f) ast_f.write('\n') return 0
def PrintAst(nodes, opts): # type: (List[command_t], Any) -> None if len(nodes) == 1: node = nodes[0] else: node = command.CommandList(nodes) if opts.ast_format == 'none': print('AST not printed.', file=sys.stderr) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError if 'abbrev-' in opts.ast_format: tree = node.AbbreviatedTree() else: tree = node.PrettyTree() ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n')
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True): # type: (cmd_value__Argv, bool, bool) -> int arg0 = cmd_val.argv[0] builtin_id = consts.LookupSpecialBuiltin(arg0) if builtin_id != consts.NO_INDEX: return self.RunBuiltin(builtin_id, cmd_val) builtin_id = consts.LookupNormalBuiltin(arg0) if builtin_id != consts.NO_INDEX: return self.RunBuiltin(builtin_id, cmd_val) log('Unhandled SimpleCommand') f = mylib.Stdout() #ast_f = fmt.DetectConsoleOutput(f) # Stupid Eclipse debugger doesn't display ANSI ast_f = fmt.TextOutput(f) tree = cmd_val.PrettyTree() ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') return 0
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') if action == 'py': # Prints the module schema_path = argv[2] with open(schema_path) as f: module = asdl.parse(f) app_types = {'id': asdl.UserType(Id)} type_lookup = asdl.ResolveTypes(module, app_types) # Note this is a big tree. But we really want a graph of pointers to # instances. # Type(name, Product(...)) # Type(name, Sum([Constructor(...), ...])) #print(module) root = sys.modules[__name__] # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a # demo. py_meta.MakeTypes(module, root, type_lookup) print('Dynamically created a Python module with these types:') for name in dir(root): print('\t' + name) elif action == 'arith-encode': # oheap encoding expr = argv[2] out_path = argv[3] obj = arith_parse.ParseShell(expr) print('Encoding %r into binary:' % expr) print(obj) enc = encode.Params() with open(out_path, 'wb') as f: out = encode.BinOutput(f) encode.EncodeRoot(obj, enc, out) elif action == 'arith-format': # pretty printing expr = argv[2] obj = arith_parse.ParseShell(expr) #out = fmt.TextOutput(sys.stdout) tree = fmt.MakeTree(obj) #treee= ['hi', 'there', ['a', 'b'], 'c'] f = fmt.DetectConsoleOutput(sys.stdout) fmt.PrintTree(tree, f) print() # Might need to print the output? # out.WriteToFile? else: raise RuntimeError('Invalid action %r' % action)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True): # type: (cmd_value__Argv, bool, bool) -> int argv = cmd_val.argv span_id = cmd_val.arg_spids[0] if len( cmd_val.arg_spids) else runtime.NO_SPID arg0 = argv[0] builtin_id = consts.LookupSpecialBuiltin(arg0) if builtin_id != consts.NO_INDEX: return self.RunBuiltin(builtin_id, cmd_val) # Copied from core/executor.py if call_procs: proc_node = self.procs.get(arg0) if proc_node is not None: if (self.exec_opts.strict_errexit() and self.mutable_opts.ErrExitIsDisabled()): # TODO: make errfmt a member #self.errfmt.Print_('errexit was disabled for this construct', # span_id=self.mutable_opts.errexit.spid_stack[0]) #stderr_line('') e_die( "Can't run a proc while errexit is disabled. " "Use 'catch' or wrap it in a process with $0 myproc", span_id=span_id) # NOTE: Functions could call 'exit 42' directly, etc. status = self.cmd_ev.RunProc(proc_node, argv[1:]) return status builtin_id = consts.LookupNormalBuiltin(arg0) if builtin_id != consts.NO_INDEX: return self.RunBuiltin(builtin_id, cmd_val) # See how many tests will pass #if mylib.PYTHON: if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling import subprocess try: status = subprocess.call(cmd_val.argv) except OSError as e: log('Error running %s: %s', cmd_val.argv, e) return 1 return status log('Unhandled SimpleCommand') f = mylib.Stdout() #ast_f = fmt.DetectConsoleOutput(f) # Stupid Eclipse debugger doesn't display ANSI ast_f = fmt.TextOutput(f) tree = cmd_val.PrettyTree() ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') return 0
def __repr__(self): # TODO: Break this circular dependency. from asdl import format as fmt ast_f = fmt.TextOutput(util.Buffer()) # No color by default. tree = fmt.MakeTree(self) fmt.PrintTree(tree, ast_f) s, _ = ast_f.GetRaw() return s
def PrettyPrint(self, f=sys.stdout): # type: (IO[str]) -> None """Print abbreviated tree in color, for debugging.""" from asdl import format as fmt ast_f = fmt.DetectConsoleOutput(f) tree = self.AbbreviatedTree() fmt.PrintTree(tree, ast_f)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True): # type: (cmd_value__Argv, bool, bool) -> int argv = cmd_val.argv span_id = cmd_val.arg_spids[0] if len( cmd_val.arg_spids) else runtime.NO_SPID arg0 = argv[0] builtin_id = consts.LookupSpecialBuiltin(arg0) if builtin_id != consts.NO_INDEX: return self.RunBuiltin(builtin_id, cmd_val) func_node = self.procs.get(arg0) if func_node is not None: if (self.exec_opts.strict_errexit() and self.mutable_opts.errexit.SpidIfDisabled() != runtime.NO_SPID): # NOTE: This would be checked below, but this gives a better error # message. e_die( "can't disable errexit running a function. " "Maybe wrap the function in a process with the at-splice " "pattern.", span_id=span_id) # NOTE: Functions could call 'exit 42' directly, etc. status = self.cmd_ev.RunProc(func_node, argv[1:]) return status builtin_id = consts.LookupNormalBuiltin(arg0) if builtin_id != consts.NO_INDEX: return self.RunBuiltin(builtin_id, cmd_val) # See how many tests will pass #if mylib.PYTHON: if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling import subprocess try: status = subprocess.call(cmd_val.argv) except OSError as e: log('Error running %s: %s', cmd_val.argv, e) return 1 return status log('Unhandled SimpleCommand') f = mylib.Stdout() #ast_f = fmt.DetectConsoleOutput(f) # Stupid Eclipse debugger doesn't display ANSI ast_f = fmt.TextOutput(f) tree = cmd_val.PrettyTree() ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') return 0
def PrettyPrint(self, f=None): # type: (Optional[mylib.Writer]) -> None """Print abbreviated tree in color, for debugging.""" from asdl import format as fmt f = f if f else mylib.Stdout() ast_f = fmt.DetectConsoleOutput(f) tree = self.AbbreviatedTree() fmt.PrintTree(tree, ast_f)
def __repr__(self): # type: () -> str # TODO: Break this circular dependency. from asdl import format as fmt ast_f = fmt.TextOutput(StringIO()) # No color by default. tree = self.PrettyTree() fmt.PrintTree(tree, ast_f) s, _ = ast_f.GetRaw() return s
def testRepeatedString(self): node = demo_asdl.assign('declare', ['-r', '-x']) f = cStringIO.StringIO() f1 = fmt.TextOutput(f) f2 = fmt.HtmlOutput(f) for ast_f in [f1, f2]: tree = node.PrettyTree() fmt.PrintTree(tree, ast_f) pretty_str = f.getvalue() print(pretty_str) if ast_f is f1: self.assertEqual('(assign name:declare flags:[-r -x])', pretty_str) t2 = node.AbbreviatedTree() fmt.PrintTree(t2, ast_f)
def testRepeatedString(self): node = arith_ast.assign('declare', ['-r', '-x']) f = cStringIO.StringIO() ast_f = fmt.TextOutput(f) tree = fmt.MakeTree(node) #print(tree) fmt.PrintTree(tree, ast_f) pretty_str = f.getvalue() print(pretty_str) self.assertEqual('(assign name:declare flags:[-r -x])', pretty_str)
def testSimpleSum(self): node = arith_ast.op_id_e.Plus print(node) f = cStringIO.StringIO() ast_f = fmt.TextOutput(f) tree = fmt.MakeTree(node) fmt.PrintTree(tree, ast_f) # Hm this prints 'Plus'. Doesn't print the class or the number. # But those aren't intrinsic. These are mostly used for ther IDENTITY. # I think the ASDL_TYPE field contains the relevant info. Yes! pretty_str = f.getvalue() print(pretty_str)
def run_tests(): # type: () -> None lex = Lexer('abc') while True: tok_type, tok_val = lex.Read() if tok_type == tok_e.Eof: break #print('%s %s' % (tok_type, tok_val)) log('tok_val %s', tok_val) CASES = [ '1+2', '1+2*3', '1*2+3', '(1+2)*3', 'a+b+c+d', 'a*b*3*4', # expect errors here: '(', ')', '(a+b', ' ', ' $$ ', ] for expr in CASES: lex = Lexer(expr) p = Parser(lex) log('') log('--') log('%s =>', expr) tree = None # type: Optional[expr_t] try: tree = p.Parse() except ParseError as e: log('Parse error: %s', e.msg) continue #log('%s', tree) htree = tree.AbbreviatedTree() ast_f = fmt.AnsiOutput(mylib.Stdout()) fmt.PrintTree(htree, ast_f) ast_f.write('\n')
def main(argv): try: action = argv[1] except IndexError: raise RuntimeError('Action required') if action == 'py': schema_path = argv[2] module = asdl.parse(schema_path) root = sys.modules[__name__] # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a # demo. py_meta.MakeTypes(module, root, app_types={'id': asdl.UserType(Id)}) print(dir(root)) elif action == 'arith-encode': expr = argv[2] out_path = argv[3] obj = arith_parse.ParseShell(expr) print('Encoding %r into binary:' % expr) print(obj) enc = encode.Params() with open(out_path, 'wb') as f: out = encode.BinOutput(f) encode.EncodeRoot(obj, enc, out) elif action == 'arith-format': expr = argv[2] obj = arith_parse.ParseShell(expr) #out = fmt.TextOutput(sys.stdout) tree = fmt.MakeTree(obj) #treee= ['hi', 'there', ['a', 'b'], 'c'] fmt.PrintTree(tree, sys.stdout) # Might need to print the output? # out.WriteToFile? else: raise RuntimeError('Invalid action %r' % action)
def PrintAst(nodes, opts): # type: (List[command_t], Any) -> None if len(nodes) == 1: node = nodes[0] else: node = command.CommandList(nodes) if opts.ast_format == 'none': print('AST not printed.', file=sys.stderr) if 0: from _devbuild.gen.id_kind_asdl import Id_str from frontend.lexer import ID_HIST for id_, count in ID_HIST.most_common(10): print('%8d %s' % (count, Id_str(id_))) print() total = sum(ID_HIST.values()) print('%8d total tokens returned' % total) else: # text output f = mylib.Stdout() if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError() if 'abbrev-' in opts.ast_format: tree = node.AbbreviatedTree() else: tree = node.PrettyTree() ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n')
def OshMain(argv0, argv, login_shell): spec = args.FlagsAndOptions() spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string spec.ShortFlag('-i') # interactive # TODO: -h too spec.LongFlag('--help') spec.LongFlag('--version') spec.LongFlag( '--ast-format', ['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap', 'none'], default='abbrev-text') spec.LongFlag('--show-ast') # execute and show spec.LongFlag('--fix') spec.LongFlag('--debug-spans') # For oshc translate spec.LongFlag('--print-status') spec.LongFlag( '--trace', ['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now spec.LongFlag('--hijack-shebang') # For benchmarks/*.sh spec.LongFlag('--parser-mem-dump', args.Str) spec.LongFlag('--runtime-mem-dump', args.Str) builtin.AddOptionsToArgSpec(spec) try: opts, opt_index = spec.Parse(argv) except args.UsageError as e: util.usage(str(e)) return 2 if opts.help: loader = util.GetResourceLoader() builtin.Help(['osh-usage'], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 trace_state = util.TraceState() if 'cmd-parse' == opts.trace: util.WrapMethods(cmd_parse.CommandParser, trace_state) if 'word-parse' == opts.trace: util.WrapMethods(word_parse.WordParser, trace_state) if 'lexer' == opts.trace: util.WrapMethods(lexer.Lexer, trace_state) if opt_index == len(argv): dollar0 = argv0 else: dollar0 = argv[opt_index] # the script name, or the arg after -c # TODO: Create a --parse action or 'osh parse' or 'oil osh-parse' # osh-fix # It uses a different memory-management model. It's a batch program and not # an interactive program. pool = alloc.Pool() arena = pool.NewArena() # TODO: Maybe wrap this initialization sequence up in an oil_State, like # lua_State. status_lines = ui.MakeStatusLines() mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ, arena) funcs = {} # Passed to Executor for 'complete', and passed to completion.Init if completion: comp_lookup = completion.CompletionLookup() else: # TODO: NullLookup? comp_lookup = None exec_opts = state.ExecOpts(mem) builtin.SetExecOpts(exec_opts, opts.opt_changes) fd_state = process.FdState() ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, readline, completion, comp_lookup, exec_opts, arena) # NOTE: The rc file can contain both commands and functions... ideally we # would only want to save nodes/lines for the functions. try: rc_path = 'oilrc' arena.PushSource(rc_path) with open(rc_path) as f: rc_line_reader = reader.FileLineReader(f, arena) _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena) try: rc_node = rc_c_parser.ParseWholeFile() if not rc_node: err = rc_c_parser.Error() ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 finally: arena.PopSource() status = ex.Execute(rc_node) #print('oilrc:', status, cflow, file=sys.stderr) # Ignore bad status? except IOError as e: if e.errno != errno.ENOENT: raise if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena) if opts.i: # -c and -i can be combined exec_opts.interactive = True elif opts.i: # force interactive arena.PushSource('<stdin -i>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena) exec_opts.interactive = True else: try: script_name = argv[opt_index] except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena) exec_opts.interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) else: arena.PushSource(script_name) try: f = fd_state.Open(script_name) except OSError as e: util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena) # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. w_parser, c_parser = parse_lib.MakeParser(line_reader, arena) if exec_opts.interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if HAVE_READLINE: splitter = legacy.SplitContext(mem) ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter) status_out = completion.StatusOutput(status_lines, exec_opts) completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup, status_out, ev) return InteractiveLoop(opts, ex, c_parser, w_parser, line_reader) else: # Parse the whole thing up front #print('Parsing file') _tlog('ParseWholeFile') # TODO: Do I need ParseAndEvalLoop? How is it different than # InteractiveLoop? try: node = c_parser.ParseWholeFile() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stderr) print('parse error: %s' % e.UserErrorString(), file=sys.stderr) return 2 else: # TODO: Remove this older form of error handling. if not node: err = c_parser.Error() assert err, err # can't be empty ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 do_exec = True if opts.fix: #log('SPANS: %s', arena.spans) osh2oil.PrintAsOil(arena, node, opts.debug_spans) do_exec = False if exec_opts.noexec: do_exec = False # Do this after parsing the entire file. There could be another option to # do it before exiting runtime? if opts.parser_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % os.getpid() with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--parser-mem-dump)', input_path, opts.parser_mem_dump) # -n prints AST, --show-ast prints and executes if exec_opts.noexec or opts.show_ast: if opts.ast_format == 'none': print('AST not printed.', file=sys.stderr) elif opts.ast_format == 'oheap': # TODO: Make this a separate flag? if sys.stdout.isatty(): raise RuntimeError( 'ERROR: Not dumping binary data to a TTY.') f = sys.stdout enc = encode.Params() out = encode.BinOutput(f) encode.EncodeRoot(node, enc, out) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError abbrev_hook = (ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None) tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook) ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') #util.log("Execution skipped because 'noexec' is on ") status = 0 if do_exec: _tlog('Execute(node)') status = ex.ExecuteAndRunExitTrap(node) # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub # don't because they call sys.exit(). if opts.runtime_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % os.getpid() with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--runtime-mem-dump)', input_path, opts.runtime_mem_dump) else: status = 0 return status
def _PrettyPrint(n): """Prints in color.""" ast_f = fmt.DetectConsoleOutput(sys.stdout) tree = n.PrettyTree() fmt.PrintTree(tree, ast_f)
def main(argv): # type: (List[str]) -> int arena = alloc.Arena() opt_array = [False] * option_i.ARRAY_SIZE parse_opts = optview.Parse(opt_array) # Dummy value; not respecting aliases! aliases = {} # type: Dict[str, str] # parse `` and a[x+1]=bar differently oil_grammar = None # type: Grammar if mylib.PYTHON: loader = pyutil.GetResourceLoader() oil_grammar = pyutil.LoadOilGrammar(loader) parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar) pretty_print = True if len(argv) == 1: line_reader = reader.FileLineReader(mylib.Stdin(), arena) src = source.Stdin('') # type: source_t elif len(argv) == 2: path = argv[1] f = mylib.open(path) line_reader = reader.FileLineReader(f, arena) src = source.MainFile(path) elif len(argv) == 3: if argv[1] == '-c': # This path is easier to run through GDB line_reader = reader.StringLineReader(argv[2], arena) src = source.CFlag() elif argv[1] == '-n': # For benchmarking, allow osh_parse -n file.txt path = argv[2] f = mylib.open(path) line_reader = reader.FileLineReader(f, arena) src = source.MainFile(path) # This is like --ast-format none, which benchmarks/osh-helper.sh passes. pretty_print = False else: raise AssertionError() else: raise AssertionError() arena.PushSource(src) c_parser = parse_ctx.MakeOshParser(line_reader) try: #node = main_loop.ParseWholeFile(c_parser) node = ParseWholeFile(c_parser) except error.Parse as e: ui.PrettyPrintError(e, arena) return 2 assert node is not None # C++ doesn't have the abbreviations yet (though there are some differences # like omitting spids) #tree = node.AbbreviatedTree() if pretty_print: tree = node.PrettyTree() ast_f = fmt.DetectConsoleOutput(mylib.Stdout()) fmt.PrintTree(tree, ast_f) ast_f.write('\n') return 0
def _PrettyPrint(n): from asdl import format as fmt ast_f = fmt.DetectConsoleOutput(sys.stdout) tree = fmt.MakeTree(n) fmt.PrintTree(tree, ast_f)
def run_tests(): # type: () -> None lex = Lexer('abc') while True: tok_type, tok_val = lex.Read() if tok_type == tok_e.Eof: break #print('%s %s' % (tok_type, tok_val)) log('tok_val %s', tok_val) CASES = [ '1+2', '1+2*3', '1*2+3', '(1+2)*3', 'a+b+c+d', 'a*b*3*4', '1', 'a', # expect errors here: '(', ')', '(a+b', ' ', ' $$ ', ] for expr in CASES: lex = Lexer(expr) p = Parser(lex) log('') log('--') log('%s =>', expr) node = None # type: Optional[expr_t] try: node = p.Parse() except ParseError as e: log('Parse error: %s', e.msg) continue #log('%s', tree) htree = node.AbbreviatedTree() ast_f = fmt.AnsiOutput(mylib.Stdout()) fmt.PrintTree(htree, ast_f) ast_f.write('\n') UP_node = node with tagswitch(UP_node) as case: if case(expr_e.Const): node = cast(expr__Const, UP_node) log('Const %d', node.i) elif case(expr_e.Var): node = cast(expr__Var, UP_node) log('Var %s', node.name) else: log('Other')
def OshMain(argv, login_shell): spec = args.FlagsAndOptions() spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string spec.ShortFlag('-i') # interactive # TODO: -h too spec.LongFlag('--help') spec.LongFlag('--version') spec.LongFlag('--ast-format', ['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap'], default='abbrev-text') spec.LongFlag('--show-ast') # execute and show spec.LongFlag('--fix') spec.LongFlag('--debug-spans') spec.LongFlag('--print-status') spec.LongFlag( '--trace', ['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now spec.LongFlag('--hijack-shebang') builtin.AddOptionsToArgSpec(spec) try: opts, opt_index = spec.Parse(argv) except args.UsageError as e: util.usage(str(e)) return 2 if opts.help: loader = util.GetResourceLoader() # TOOD: Use Global builtin.Help(['osh-usage'], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 trace_state = util.TraceState() if 'cmd-parse' == opts.trace: util.WrapMethods(cmd_parse.CommandParser, trace_state) if 'word-parse' == opts.trace: util.WrapMethods(word_parse.WordParser, trace_state) if 'lexer' == opts.trace: util.WrapMethods(lexer.Lexer, trace_state) if opt_index == len(argv): dollar0 = sys.argv[0] # e.g. bin/osh else: dollar0 = argv[opt_index] # the script name, or the arg after -c # TODO: Create a --parse action or 'osh parse' or 'oil osh-parse' # osh-fix # It uses a different memory-management model. It's a batch program and not # an interactive program. pool = alloc.Pool() arena = pool.NewArena() # TODO: Maybe wrap this initialization sequence up in an oil_State, like # lua_State. status_lines = ui.MakeStatusLines() mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ) funcs = {} # Passed to Executor for 'complete', and passed to completion.Init if completion: comp_lookup = completion.CompletionLookup() else: # TODO: NullLookup? comp_lookup = None exec_opts = state.ExecOpts() builtin.SetExecOpts(exec_opts, opts.opt_changes) # TODO: How to get a handle to initialized builtins here? # tokens.py has it. I think you just make a separate table, with # metaprogramming. ex = cmd_exec.Executor(mem, status_lines, funcs, completion, comp_lookup, exec_opts, arena) # NOTE: The rc file can contain both commands and functions... ideally we # would only want to save nodes/lines for the functions. try: rc_path = 'oilrc' arena.PushSource(rc_path) with open(rc_path) as f: rc_line_reader = reader.FileLineReader(f, arena=arena) _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena) try: rc_node = rc_c_parser.ParseWholeFile() if not rc_node: # TODO: Error should return a token, and then the token should have a # arena index, and then look that up in the arena. err = rc_c_parser.Error() ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 finally: arena.PopSource() status = ex.Execute(rc_node) #print('oilrc:', status, cflow, file=sys.stderr) # Ignore bad status? except IOError as e: if e.errno != errno.ENOENT: raise if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena=arena) interactive = False elif opts.i: # force interactive arena.PushSource('<stdin -i>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena) interactive = True else: try: script_name = argv[opt_index] except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena) interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena=arena) interactive = False else: arena.PushSource(script_name) # TODO: Does this open file descriptor need to be moved beyond 3..9 ? # Yes! See dash input.c setinputfile. It calls savefd(). # TODO: It also needs to be closed later. try: f = open(script_name) except IOError as e: util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena=arena) interactive = False # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. w_parser, c_parser = parse_lib.MakeParser(line_reader, arena) if interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if HAVE_READLINE: ev = word_eval.CompletionWordEvaluator(mem, exec_opts) status_out = completion.StatusOutput(status_lines, exec_opts) completion.Init(builtin.BUILTIN_DEF, mem, funcs, comp_lookup, status_out, ev) # TODO: Could instantiate "printer" instead of showing ops InteractiveLoop(opts, ex, c_parser, w_parser, line_reader) status = 0 # TODO: set code else: # Parse the whole thing up front #print('Parsing file') tlog('ParseWholeFile') # TODO: Do I need ParseAndEvalLoop? How is it different than # InteractiveLoop? try: node = c_parser.ParseWholeFile() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stderr) print('parse error: %s' % e.UserErrorString(), file=sys.stderr) return 2 else: # TODO: Remove this older form of error handling. if not node: err = c_parser.Error() ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 do_exec = True if opts.fix: osh2oil.PrintAsOil(arena, node, opts.debug_spans) do_exec = False if exec_opts.noexec: do_exec = False if exec_opts.noexec or opts.show_ast: # -n shows the AST if opts.ast_format == 'oheap': # TODO: Make this a separate flag? if sys.stdout.isatty(): raise RuntimeError( 'ERROR: Not dumping binary data to a TTY.') f = sys.stdout enc = encode.Params() out = encode.BinOutput(f) encode.EncodeRoot(node, enc, out) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError abbrev_hook = (ast.AbbreviateNodes if 'abbrev-' in opts.ast_format else None) tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook) ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') #util.log("Execution skipped because 'noexec' is on ") status = 0 if do_exec: tlog('Execute(node)') status = ex.Execute(node) else: status = 0 return status