def OshMain(argv0, argv, login_shell): spec = args.FlagsAndOptions() spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string spec.ShortFlag('-i') # interactive # TODO: -h too spec.LongFlag('--help') spec.LongFlag('--version') spec.LongFlag( '--ast-format', ['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap', 'none'], default='abbrev-text') spec.LongFlag('--show-ast') # execute and show spec.LongFlag('--fix') spec.LongFlag('--debug-spans') # For oshc translate spec.LongFlag('--print-status') spec.LongFlag( '--trace', ['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now spec.LongFlag('--hijack-shebang') # For benchmarks/*.sh spec.LongFlag('--parser-mem-dump', args.Str) spec.LongFlag('--runtime-mem-dump', args.Str) builtin.AddOptionsToArgSpec(spec) try: opts, opt_index = spec.Parse(argv) except args.UsageError as e: util.usage(str(e)) return 2 if opts.help: loader = util.GetResourceLoader() builtin.Help(['osh-usage'], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 trace_state = util.TraceState() if 'cmd-parse' == opts.trace: util.WrapMethods(cmd_parse.CommandParser, trace_state) if 'word-parse' == opts.trace: util.WrapMethods(word_parse.WordParser, trace_state) if 'lexer' == opts.trace: util.WrapMethods(lexer.Lexer, trace_state) if opt_index == len(argv): dollar0 = argv0 else: dollar0 = argv[opt_index] # the script name, or the arg after -c # TODO: Create a --parse action or 'osh parse' or 'oil osh-parse' # osh-fix # It uses a different memory-management model. It's a batch program and not # an interactive program. pool = alloc.Pool() arena = pool.NewArena() # TODO: Maybe wrap this initialization sequence up in an oil_State, like # lua_State. status_lines = ui.MakeStatusLines() mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ, arena) funcs = {} # Passed to Executor for 'complete', and passed to completion.Init if completion: comp_lookup = completion.CompletionLookup() else: # TODO: NullLookup? comp_lookup = None exec_opts = state.ExecOpts(mem) builtin.SetExecOpts(exec_opts, opts.opt_changes) fd_state = process.FdState() ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, readline, completion, comp_lookup, exec_opts, arena) # NOTE: The rc file can contain both commands and functions... ideally we # would only want to save nodes/lines for the functions. try: rc_path = 'oilrc' arena.PushSource(rc_path) with open(rc_path) as f: rc_line_reader = reader.FileLineReader(f, arena) _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena) try: rc_node = rc_c_parser.ParseWholeFile() if not rc_node: err = rc_c_parser.Error() ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 finally: arena.PopSource() status = ex.Execute(rc_node) #print('oilrc:', status, cflow, file=sys.stderr) # Ignore bad status? except IOError as e: if e.errno != errno.ENOENT: raise if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena) if opts.i: # -c and -i can be combined exec_opts.interactive = True elif opts.i: # force interactive arena.PushSource('<stdin -i>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena) exec_opts.interactive = True else: try: script_name = argv[opt_index] except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena) exec_opts.interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) else: arena.PushSource(script_name) try: f = fd_state.Open(script_name) except OSError as e: util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena) # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. w_parser, c_parser = parse_lib.MakeParser(line_reader, arena) if exec_opts.interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if HAVE_READLINE: splitter = legacy.SplitContext(mem) ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter) status_out = completion.StatusOutput(status_lines, exec_opts) completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup, status_out, ev) return InteractiveLoop(opts, ex, c_parser, w_parser, line_reader) else: # Parse the whole thing up front #print('Parsing file') _tlog('ParseWholeFile') # TODO: Do I need ParseAndEvalLoop? How is it different than # InteractiveLoop? try: node = c_parser.ParseWholeFile() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stderr) print('parse error: %s' % e.UserErrorString(), file=sys.stderr) return 2 else: # TODO: Remove this older form of error handling. if not node: err = c_parser.Error() assert err, err # can't be empty ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 do_exec = True if opts.fix: #log('SPANS: %s', arena.spans) osh2oil.PrintAsOil(arena, node, opts.debug_spans) do_exec = False if exec_opts.noexec: do_exec = False # Do this after parsing the entire file. There could be another option to # do it before exiting runtime? if opts.parser_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % os.getpid() with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--parser-mem-dump)', input_path, opts.parser_mem_dump) # -n prints AST, --show-ast prints and executes if exec_opts.noexec or opts.show_ast: if opts.ast_format == 'none': print('AST not printed.', file=sys.stderr) elif opts.ast_format == 'oheap': # TODO: Make this a separate flag? if sys.stdout.isatty(): raise RuntimeError( 'ERROR: Not dumping binary data to a TTY.') f = sys.stdout enc = encode.Params() out = encode.BinOutput(f) encode.EncodeRoot(node, enc, out) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError abbrev_hook = (ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None) tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook) ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') #util.log("Execution skipped because 'noexec' is on ") status = 0 if do_exec: _tlog('Execute(node)') status = ex.ExecuteAndRunExitTrap(node) # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub # don't because they call sys.exit(). if opts.runtime_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % os.getpid() with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--runtime-mem-dump)', input_path, opts.runtime_mem_dump) else: status = 0 return status
def OshCommandMain(argv): """Run an 'oshc' tool. 'osh' is short for "osh compiler" or "osh command". TODO: - oshc --help oshc deps --path: the $PATH to use to find executables. What about libraries? NOTE: we're leaving out su -c, find, xargs, etc.? Those should generally run functions using the $0 pattern. --chained-command sudo """ try: action = argv[0] except IndexError: raise args.UsageError('oshc: Missing required subcommand.') if action not in SUBCOMMANDS: raise args.UsageError('oshc: Invalid subcommand %r.' % action) try: script_name = argv[1] except IndexError: script_name = '<stdin>' f = sys.stdin else: try: f = open(script_name) except IOError as e: util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno)) return 2 pool = alloc.Pool() arena = pool.NewArena() arena.PushSource(script_name) line_reader = reader.FileLineReader(f, arena) _, c_parser = parse_lib.MakeParser(line_reader, arena) try: node = c_parser.ParseWholeFile() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stderr) print('parse error: %s' % e.UserErrorString(), file=sys.stderr) return 2 else: # TODO: Remove this older form of error handling. if not node: err = c_parser.Error() assert err, err # can't be empty ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 f.close() # Columns for list-* # path line name # where name is the binary path, variable name, or library path. # bin-deps and lib-deps can be used to make an app bundle. # Maybe I should list them together? 'deps' can show 4 columns? # # path, line, type, name # # --pretty can show the LST location. # stderr: show how we're following imports? if action == 'translate': # TODO: FIx this invocation up. #debug_spans = opt.debug_spans debug_spans = False osh2oil.PrintAsOil(arena, node, debug_spans) elif action == 'format': # TODO: autoformat code raise NotImplementedError(action) elif action == 'deps': deps.Deps(node) elif action == 'undefined-vars': # could be environment variables pass else: raise AssertionError # Checked above return 0
def OshCommandMain(argv): """Run an 'oshc' tool. 'osh' is short for "osh compiler" or "osh command". TODO: - oshc --help oshc deps --path: the $PATH to use to find executables. What about libraries? NOTE: we're leaving out su -c, find, xargs, etc.? Those should generally run functions using the $0 pattern. --chained-command sudo """ try: action = argv[0] except IndexError: raise args.UsageError('oshc: Missing required subcommand.') if action not in SUBCOMMANDS: raise args.UsageError('oshc: Invalid subcommand %r.' % action) try: script_name = argv[1] except IndexError: script_name = '<stdin>' f = sys.stdin else: try: f = open(script_name) except IOError as e: util.error("Couldn't open %r: %s", script_name, posix.strerror(e.errno)) return 2 pool = alloc.Pool() arena = pool.NewArena() arena.PushSource(script_name) line_reader = reader.FileLineReader(f, arena) aliases = {} # Dummy value; not respecting aliases! parse_ctx = parse_lib.ParseContext(arena, aliases) c_parser = parse_ctx.MakeOshParser(line_reader) try: node = main_loop.ParseWholeFile(c_parser) except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stderr) return 2 assert node is not None f.close() # Columns for list-* # path line name # where name is the binary path, variable name, or library path. # bin-deps and lib-deps can be used to make an app bundle. # Maybe I should list them together? 'deps' can show 4 columns? # # path, line, type, name # # --pretty can show the LST location. # stderr: show how we're following imports? if action == 'translate': osh2oil.PrintAsOil(arena, node) elif action == 'arena': # for debugging osh2oil.PrintArena(arena) elif action == 'spans': # for debugging osh2oil.PrintSpans(arena) elif action == 'format': # TODO: autoformat code raise NotImplementedError(action) elif action == 'deps': deps.Deps(node) elif action == 'undefined-vars': # could be environment variables raise NotImplementedError else: raise AssertionError # Checked above return 0
def _InitWordParserWithArena(s): pool = alloc.Pool() arena = pool.NewArena() line_reader, lexer = parse_lib.InitLexer(s, arena=arena) w_parser = WordParser(lexer, line_reader) return arena, w_parser
def OshMain(argv, login_shell): spec = args.FlagsAndOptions() spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string spec.ShortFlag('-i') # interactive # TODO: -h too spec.LongFlag('--help') spec.LongFlag('--version') spec.LongFlag('--ast-format', ['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap'], default='abbrev-text') spec.LongFlag('--show-ast') # execute and show spec.LongFlag('--fix') spec.LongFlag('--debug-spans') spec.LongFlag('--print-status') spec.LongFlag( '--trace', ['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now spec.LongFlag('--hijack-shebang') builtin.AddOptionsToArgSpec(spec) try: opts, opt_index = spec.Parse(argv) except args.UsageError as e: util.usage(str(e)) return 2 if opts.help: loader = util.GetResourceLoader() # TOOD: Use Global builtin.Help(['osh-usage'], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 trace_state = util.TraceState() if 'cmd-parse' == opts.trace: util.WrapMethods(cmd_parse.CommandParser, trace_state) if 'word-parse' == opts.trace: util.WrapMethods(word_parse.WordParser, trace_state) if 'lexer' == opts.trace: util.WrapMethods(lexer.Lexer, trace_state) if opt_index == len(argv): dollar0 = sys.argv[0] # e.g. bin/osh else: dollar0 = argv[opt_index] # the script name, or the arg after -c # TODO: Create a --parse action or 'osh parse' or 'oil osh-parse' # osh-fix # It uses a different memory-management model. It's a batch program and not # an interactive program. pool = alloc.Pool() arena = pool.NewArena() # TODO: Maybe wrap this initialization sequence up in an oil_State, like # lua_State. status_lines = ui.MakeStatusLines() mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ) funcs = {} # Passed to Executor for 'complete', and passed to completion.Init if completion: comp_lookup = completion.CompletionLookup() else: # TODO: NullLookup? comp_lookup = None exec_opts = state.ExecOpts() builtin.SetExecOpts(exec_opts, opts.opt_changes) # TODO: How to get a handle to initialized builtins here? # tokens.py has it. I think you just make a separate table, with # metaprogramming. ex = cmd_exec.Executor(mem, status_lines, funcs, completion, comp_lookup, exec_opts, arena) # NOTE: The rc file can contain both commands and functions... ideally we # would only want to save nodes/lines for the functions. try: rc_path = 'oilrc' arena.PushSource(rc_path) with open(rc_path) as f: rc_line_reader = reader.FileLineReader(f, arena=arena) _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena) try: rc_node = rc_c_parser.ParseWholeFile() if not rc_node: # TODO: Error should return a token, and then the token should have a # arena index, and then look that up in the arena. err = rc_c_parser.Error() ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 finally: arena.PopSource() status = ex.Execute(rc_node) #print('oilrc:', status, cflow, file=sys.stderr) # Ignore bad status? except IOError as e: if e.errno != errno.ENOENT: raise if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena=arena) interactive = False elif opts.i: # force interactive arena.PushSource('<stdin -i>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena) interactive = True else: try: script_name = argv[opt_index] except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena) interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena=arena) interactive = False else: arena.PushSource(script_name) # TODO: Does this open file descriptor need to be moved beyond 3..9 ? # Yes! See dash input.c setinputfile. It calls savefd(). # TODO: It also needs to be closed later. try: f = open(script_name) except IOError as e: util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena=arena) interactive = False # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. w_parser, c_parser = parse_lib.MakeParser(line_reader, arena) if interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if HAVE_READLINE: ev = word_eval.CompletionWordEvaluator(mem, exec_opts) status_out = completion.StatusOutput(status_lines, exec_opts) completion.Init(builtin.BUILTIN_DEF, mem, funcs, comp_lookup, status_out, ev) # TODO: Could instantiate "printer" instead of showing ops InteractiveLoop(opts, ex, c_parser, w_parser, line_reader) status = 0 # TODO: set code else: # Parse the whole thing up front #print('Parsing file') tlog('ParseWholeFile') # TODO: Do I need ParseAndEvalLoop? How is it different than # InteractiveLoop? try: node = c_parser.ParseWholeFile() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stderr) print('parse error: %s' % e.UserErrorString(), file=sys.stderr) return 2 else: # TODO: Remove this older form of error handling. if not node: err = c_parser.Error() ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 do_exec = True if opts.fix: osh2oil.PrintAsOil(arena, node, opts.debug_spans) do_exec = False if exec_opts.noexec: do_exec = False if exec_opts.noexec or opts.show_ast: # -n shows the AST if opts.ast_format == 'oheap': # TODO: Make this a separate flag? if sys.stdout.isatty(): raise RuntimeError( 'ERROR: Not dumping binary data to a TTY.') f = sys.stdout enc = encode.Params() out = encode.BinOutput(f) encode.EncodeRoot(node, enc, out) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError abbrev_hook = (ast.AbbreviateNodes if 'abbrev-' in opts.ast_format else None) tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook) ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') #util.log("Execution skipped because 'noexec' is on ") status = 0 if do_exec: tlog('Execute(node)') status = ex.Execute(node) else: status = 0 return status
def ShellMain(lang, argv0, argv, login_shell): """Used by bin/osh and bin/oil. Args: lang: 'osh' or 'oil' argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox. login_shell: Was - on the front? """ # Differences between osh and oil: # - --help? I guess Oil has a SUPERSET of OSH options. # - oshrc vs oilrc # - the parser and executor # - Change the prompt in the interactive shell? assert lang in ('osh', 'oil'), lang arg_r = args.Reader(argv) try: opts = OSH_SPEC.Parse(arg_r) except args.UsageError as e: ui.usage('osh usage error: %s', e) return 2 if opts.help: loader = util.GetResourceLoader() builtin.Help(['%s-usage' % lang], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 # TODO: This should be in interactive mode only? builtin.RegisterSigIntHandler() if arg_r.AtEnd(): dollar0 = argv0 has_main = False else: dollar0 = arg_r.Peek() # the script name, or the arg after -c has_main = True pool = alloc.Pool() arena = pool.NewArena() # NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a # required arg. mem = state.Mem(dollar0, argv[arg_r.i + 1:], posix.environ, arena, has_main=has_main) funcs = {} fd_state = process.FdState() exec_opts = state.ExecOpts(mem, readline) builtin.SetExecOpts(exec_opts, opts.opt_changes) aliases = {} # feedback between runtime and parser parse_ctx = parse_lib.ParseContext(arena, aliases) # For main_loop # Three ParseContext instances SHARE aliases. TODO: Complete aliases. comp_arena = pool.NewArena() comp_arena.PushSource('<completion>') trail1 = parse_lib.Trail() comp_ctx = parse_lib.ParseContext(comp_arena, aliases, trail=trail1) hist_arena = pool.NewArena() hist_arena.PushSource('<history>') trail2 = parse_lib.Trail() hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2) # Deps helps manages dependencies. These dependencies are circular: # - ex and word_ev, arith_ev -- for command sub, arith sub # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 )) # - ex and builtins (which execute code, like eval) # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P exec_deps = cmd_exec.Deps() if opts.debug_file: debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w')) else: debug_f = util.NullDebugFile() exec_deps.debug_f = debug_f debug_f.log('Debug file is %s', opts.debug_file) splitter = split.SplitContext(mem) exec_deps.splitter = splitter # Controlled by env variable, flag, or hook? exec_deps.dumper = dev.CrashDumper( posix.environ.get('OSH_CRASH_DUMP_DIR', '')) if opts.xtrace_to_debug_file: trace_f = debug_f else: trace_f = util.DebugFile(sys.stderr) exec_deps.trace_f = trace_f # TODO: Separate comp_state and comp_lookup. comp_state = completion.State() comp_lookup = completion.Lookup() builtins = { # Lookup builtin_e.HISTORY: builtin.History(readline), builtin_e.COMPOPT: builtin_comp.CompOpt(comp_state), builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem), } ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts, parse_ctx, exec_deps) exec_deps.ex = ex word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena) exec_deps.word_ev = word_ev arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena) exec_deps.arith_ev = arith_ev word_ev.arith_ev = arith_ev # Another circular dependency bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena) exec_deps.bool_ev = bool_ev tracer = cmd_exec.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f) exec_deps.tracer = tracer # HACK for circular deps ex.word_ev = word_ev ex.arith_ev = arith_ev ex.bool_ev = bool_ev ex.tracer = tracer spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter) # Add some builtins that depend on the executor! complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup) # used later builtins[builtin_e.COMPLETE] = complete_builtin builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder) if lang == 'oil': # The Oil executor wraps an OSH executor? It needs to be able to source # it. ex = oil_cmd_exec.OilExecutor(ex) # PromptEvaluator rendering is needed in non-interactive shells for @P. prompt_ev = ui.PromptEvaluator(lang, arena, parse_ctx, ex, mem) exec_deps.prompt_ev = prompt_ev word_ev.prompt_ev = prompt_ev # HACK for circular deps # History evaluation is a no-op if readline is None. hist_ev = reader.HistoryEvaluator(readline, hist_ctx, debug_f) # Calculate ~/.config/oil/oshrc or oilrc # Use ~/.config/oil to avoid cluttering the user's home directory. Some # users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc. # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other home_dir = mem.GetVar('HOME') assert home_dir.tag == value_e.Str, home_dir rc_path = opts.rcfile or os_path.join(home_dir.s, '.config/oil', lang + 'rc') history_filename = os_path.join(home_dir.s, '.config/oil', 'history_' + lang) if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena) if opts.i: # -c and -i can be combined exec_opts.interactive = True elif opts.i: # force interactive arena.PushSource('<stdin -i>') # interactive shell only line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev) exec_opts.interactive = True else: try: script_name = arg_r.Peek() except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') # interactive shell only line_reader = reader.InteractiveLineReader( arena, prompt_ev, hist_ev) exec_opts.interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) else: arena.PushSource(script_name) try: f = fd_state.Open(script_name) except OSError as e: util.error("Couldn't open %r: %s", script_name, posix.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena) # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. if lang == 'osh': c_parser = parse_ctx.MakeOshParser(line_reader) else: c_parser = parse_ctx.MakeOilParser(line_reader) if exec_opts.interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if readline: ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps, arena) progress_f = ui.StatusLine() root_comp = completion.RootCompleter(ev, mem, comp_lookup, comp_state, comp_ctx, progress_f, debug_f) _InitReadline(readline, history_filename, root_comp, debug_f) _InitDefaultCompletions(ex, complete_builtin, comp_lookup) # NOTE: Call this AFTER _InitDefaultCompletions. SourceStartupFile(rc_path, lang, parse_ctx, ex) return main_loop.Interactive(opts, ex, c_parser, arena) # TODO: Remove this after removing it from benchmarks/osh-runtime. It's no # longer relevant with main_loop. if opts.parser_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--parser-mem-dump)', input_path, opts.parser_mem_dump) nodes_out = [] if exec_opts.noexec else None _tlog('Execute(node)') status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out) # Only print nodes if the whole parse succeeded. if nodes_out is not None and status == 0: ui.PrintAst(nodes_out, opts) # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub # don't because they call sys.exit(). if opts.runtime_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--runtime-mem-dump)', input_path, opts.runtime_mem_dump) # NOTE: This doesn't cause any spec tests to fail, but it could. if posix.environ.get('ASDL_TYPE_CHECK'): log('NOTE: Performed %d ASDL_TYPE_CHECKs.', runtime.NUM_TYPE_CHECKS) # NOTE: We haven't closed the file opened with fd_state.Open return status
def MakeArena(source_name): pool = alloc.Pool() arena = pool.NewArena() arena.PushSource('<lex_test.py>') return arena
def MakeArena(source_name): pool = alloc.Pool() arena = pool.NewArena() arena.PushSource(source_name) return arena
def setUp(self): self.pool = alloc.Pool()
def OshMain(argv0, argv, login_shell): arg_r = args.Reader(argv) try: opts = OSH_SPEC.Parse(arg_r) except args.UsageError as e: ui.usage('osh usage error: %s', e) return 2 if opts.help: loader = util.GetResourceLoader() builtin.Help(['osh-usage'], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 # TODO: This should be in interactive mode only? builtin.RegisterSigIntHandler() if arg_r.AtEnd(): dollar0 = argv0 has_main = False else: dollar0 = arg_r.Peek() # the script name, or the arg after -c has_main = True pool = alloc.Pool() arena = pool.NewArena() # NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a # required arg. mem = state.Mem(dollar0, argv[arg_r.i + 1:], posix.environ, arena, has_main=has_main) funcs = {} comp_lookup = completion.CompletionLookup() fd_state = process.FdState() exec_opts = state.ExecOpts(mem, readline) builtin.SetExecOpts(exec_opts, opts.opt_changes) aliases = {} # feedback between runtime and parser parse_ctx = parse_lib.ParseContext(arena, aliases) if opts.debug_file: debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w')) else: debug_f = util.NullDebugFile() debug_f.log('Debug file is %s', opts.debug_file) # Controlled by env variable, flag, or hook? dumper = dev.CrashDumper(posix.environ.get('OSH_CRASH_DUMP_DIR', '')) if opts.xtrace_to_debug_file: trace_f = debug_f else: trace_f = util.DebugFile(sys.stderr) devtools = dev.DevTools(dumper, debug_f, trace_f) ex = cmd_exec.Executor(mem, fd_state, funcs, comp_lookup, exec_opts, parse_ctx, devtools) # NOTE: The rc file can contain both commands and functions... ideally we # would only want to save nodes/lines for the functions. try: rc_path = 'oilrc' arena.PushSource(rc_path) with open(rc_path) as f: rc_line_reader = reader.FileLineReader(f, arena) _, rc_c_parser = parse_ctx.MakeParser(rc_line_reader) try: status = main_loop.Batch(ex, rc_c_parser, arena) finally: arena.PopSource() except IOError as e: if e.errno != errno.ENOENT: raise # Needed in non-interactive shells for @P prompt = ui.Prompt(arena, parse_ctx, ex) ui.PROMPT = prompt if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena) if opts.i: # -c and -i can be combined exec_opts.interactive = True elif opts.i: # force interactive arena.PushSource('<stdin -i>') line_reader = reader.InteractiveLineReader(arena, prompt) exec_opts.interactive = True else: try: script_name = arg_r.Peek() except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') line_reader = reader.InteractiveLineReader(arena, prompt) exec_opts.interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) else: arena.PushSource(script_name) try: f = fd_state.Open(script_name) except OSError as e: util.error("Couldn't open %r: %s", script_name, posix.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena) # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. w_parser, c_parser = parse_ctx.MakeParser(line_reader) if exec_opts.interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if HAVE_READLINE: splitter = legacy.SplitContext(mem) # TODO: share with executor. ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter, arena) progress_f = ui.StatusLine() var_action = completion.VariablesActionInternal(ex.mem) root_comp = completion.RootCompleter(ev, comp_lookup, var_action, parse_ctx, progress_f, debug_f) completion.Init(readline, root_comp, debug_f) _InitDefaultCompletions(ex, comp_lookup) return main_loop.Interactive(opts, ex, c_parser, arena) # TODO: Remove this after removing it from benchmarks/osh-runtime. It's no # longer relevant with main_loop. if opts.parser_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--parser-mem-dump)', input_path, opts.parser_mem_dump) nodes_out = [] if exec_opts.noexec else None _tlog('Execute(node)') status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out) if nodes_out is not None: ui.PrintAst(nodes_out, opts) # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub # don't because they call sys.exit(). if opts.runtime_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--runtime-mem-dump)', input_path, opts.runtime_mem_dump) # NOTE: We haven't closed the file opened with fd_state.Open return status
def ShellMain(lang, argv0, argv, login_shell): """Used by bin/osh and bin/oil. Args: lang: 'osh' or 'oil' argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox. login_shell: Was - on the front? """ # Differences between osh and oil: # - --help? I guess Oil has a SUPERSET of OSH options. # - oshrc vs oilrc # - the parser and executor # - Change the prompt in the interactive shell? assert lang in ('osh', 'oil'), lang arg_r = args.Reader(argv) try: opts = OSH_SPEC.Parse(arg_r) except args.UsageError as e: ui.usage('osh usage error: %s', e) return 2 # NOTE: This has a side effect of deleting _OVM_* from the environment! # TODO: Thread this throughout the program, and get rid of the global # variable in core/util.py. Rename to InitResourceLaoder(). It's now only # used for the 'help' builtin and --version. loader = pyutil.GetResourceLoader() if opts.help: builtin.Help(['%s-usage' % lang], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 if arg_r.AtEnd(): dollar0 = argv0 has_main = False else: dollar0 = arg_r.Peek() # the script name, or the arg after -c has_main = True pool = alloc.Pool() arena = pool.NewArena() # NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a # required arg. mem = state.Mem(dollar0, argv[arg_r.i + 1:], posix.environ, arena, has_main=has_main) funcs = {} fd_state = process.FdState() exec_opts = state.ExecOpts(mem, line_input) builtin.SetExecOpts(exec_opts, opts.opt_changes) aliases = {} # feedback between runtime and parser if opts.one_pass_parse and not exec_opts.noexec: raise args.UsageError('--one-pass-parse requires noexec (-n)') parse_ctx = parse_lib.ParseContext(arena, aliases, one_pass_parse=opts.one_pass_parse) # Three ParseContext instances SHARE aliases. comp_arena = pool.NewArena() comp_arena.PushSource('<completion>') trail1 = parse_lib.Trail() # one_pass_parse needs to be turned on to complete inside backticks. TODO: # fix the issue where ` gets erased because it's not part of # set_completer_delims(). comp_ctx = parse_lib.ParseContext(comp_arena, aliases, trail=trail1, one_pass_parse=True) hist_arena = pool.NewArena() hist_arena.PushSource('<history>') trail2 = parse_lib.Trail() hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2) # Deps helps manages dependencies. These dependencies are circular: # - ex and word_ev, arith_ev -- for command sub, arith sub # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 )) # - ex and builtins (which execute code, like eval) # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P exec_deps = cmd_exec.Deps() my_pid = posix.getpid() debug_path = '' debug_dir = posix.environ.get('OSH_DEBUG_DIR') if opts.debug_file: # --debug-file takes precedence over OSH_DEBUG_DIR debug_path = opts.debug_file elif debug_dir: debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid) if debug_path: # This will be created as an empty file if it doesn't exist, or it could be # a pipe. try: debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w')) except OSError as e: util.error("Couldn't open %r: %s", debug_path, posix.strerror(e.errno)) return 2 else: debug_f = util.NullDebugFile() exec_deps.debug_f = debug_f # Not using datetime for dependency reasons. TODO: maybe show the date at # the beginning of the log, and then only show time afterward? To save # space, and make space for microseconds. (datetime supports microseconds # but time.strftime doesn't). iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S") debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv) if debug_path: debug_f.log('Writing logs to %r', debug_path) interp = posix.environ.get('OSH_HIJACK_SHEBANG', '') exec_deps.ext_prog = process.ExternalProgram(interp, fd_state, debug_f) splitter = split.SplitContext(mem) exec_deps.splitter = splitter # This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be # stuffing too much into one, since a .json crash dump isn't a stream. crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '') exec_deps.dumper = dev.CrashDumper(crash_dump_dir) if opts.xtrace_to_debug_file: trace_f = debug_f else: trace_f = util.DebugFile(sys.stderr) exec_deps.trace_f = trace_f comp_lookup = completion.Lookup() # # Various Global State objects to work around readline interfaces # compopt_state = completion.OptionState() comp_ui_state = comp_ui.State() prompt_state = comp_ui.PromptState() builtins = { # Lookup builtin_e.HISTORY: builtin.History(line_input), builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state), builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem), } ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts, parse_ctx, exec_deps) exec_deps.ex = ex word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena) exec_deps.word_ev = word_ev arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena) exec_deps.arith_ev = arith_ev word_ev.arith_ev = arith_ev # Another circular dependency bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena) exec_deps.bool_ev = bool_ev tracer = cmd_exec.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f) exec_deps.tracer = tracer # HACK for circular deps ex.word_ev = word_ev ex.arith_ev = arith_ev ex.bool_ev = bool_ev ex.tracer = tracer spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter, comp_lookup) # Add some builtins that depend on the executor! complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup) builtins[builtin_e.COMPLETE] = complete_builtin builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder) if lang == 'oil': # The Oil executor wraps an OSH executor? It needs to be able to source # it. ex = oil_cmd_exec.OilExecutor(ex) # PromptEvaluator rendering is needed in non-interactive shells for @P. prompt_ev = prompt.Evaluator(lang, arena, parse_ctx, ex, mem) exec_deps.prompt_ev = prompt_ev word_ev.prompt_ev = prompt_ev # HACK for circular deps # History evaluation is a no-op if line_input is None. hist_ev = history.Evaluator(line_input, hist_ctx, debug_f) # Calculate ~/.config/oil/oshrc or oilrc # Use ~/.config/oil to avoid cluttering the user's home directory. Some # users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc. # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other home_dir = process.GetHomeDir() assert home_dir is not None rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil', lang + 'rc') history_filename = os_path.join(home_dir, '.config/oil', 'history_' + lang) if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena) if opts.i: # -c and -i can be combined exec_opts.interactive = True elif opts.i: # force interactive arena.PushSource('<stdin -i>') # interactive shell only line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev, line_input, prompt_state) exec_opts.interactive = True else: try: script_name = arg_r.Peek() except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') # interactive shell only line_reader = reader.InteractiveLineReader( arena, prompt_ev, hist_ev, line_input, prompt_state) exec_opts.interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) else: arena.PushSource(script_name) try: f = fd_state.Open(script_name) except OSError as e: util.error("Couldn't open %r: %s", script_name, posix.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena) # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. if lang == 'osh': c_parser = parse_ctx.MakeOshParser(line_reader) else: c_parser = parse_ctx.MakeOilParser(line_reader) # NOTE: SIGINT is temporarily enabled during readline() by # frontend/reader.py. # It's treated differently than SIGQUIT and SIGTSTP because Python handles it # with KeyboardInterrupt. We don't want KeyboardInterrupt at arbitrary # points in a non-interactive shell. (e.g. osh -c 'sleep 5' then Ctrl-C) signal.signal(signal.SIGINT, signal.SIG_IGN) if exec_opts.interactive: if line_input: # NOTE: We're using a different WordEvaluator here. ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps, arena) root_comp = completion.RootCompleter(ev, mem, comp_lookup, compopt_state, comp_ui_state, comp_ctx, debug_f) term_width = 0 if opts.completion_display == 'nice': try: term_width = libc.get_terminal_width() except IOError: # stdin not a terminal pass if term_width != 0: display = comp_ui.NiceDisplay(term_width, comp_ui_state, prompt_state, debug_f, line_input) else: display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state, debug_f) _InitReadline(line_input, history_filename, root_comp, display, debug_f) _InitDefaultCompletions(ex, complete_builtin, comp_lookup) else: # Without readline module display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state, debug_f) # The shell itself should ignore Ctrl-\. signal.signal(signal.SIGQUIT, signal.SIG_IGN) # This prevents Ctrl-Z from suspending OSH in interactive mode. But we're # not getting notification via wait() that the child stopped? signal.signal(signal.SIGTSTP, signal.SIG_IGN) # Register a callback to receive terminal width changes. signal.signal(signal.SIGWINCH, lambda x, y: display.OnWindowChange()) # NOTE: Call this AFTER _InitDefaultCompletions. SourceStartupFile(rc_path, lang, parse_ctx, ex) line_reader.Reset() # After sourcing startup file, render $PS1 return main_loop.Interactive(opts, ex, c_parser, display, arena) # TODO: This doesn't do anything interesting. # - Remove the column from osh-runtime, since it doesn't work with main_loop. # - http://www.oilshell.org/release/0.6.pre6/benchmarks.wwz/osh-runtime/ # - Move it to right before ui.PrintAst(), so we can use it in -n mode. # This benchmark has been broken since 0.6.pre4. # http://www.oilshell.org/release/0.6.pre3/benchmarks.wwz/osh-parser/ if opts.parser_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--parser-mem-dump)', input_path, opts.parser_mem_dump) nodes_out = [] if exec_opts.noexec else None _tlog('Execute(node)') status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out) # Only print nodes if the whole parse succeeded. if nodes_out is not None and status == 0: ui.PrintAst(nodes_out, opts) # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub # don't because they call sys.exit(). if opts.runtime_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--runtime-mem-dump)', input_path, opts.runtime_mem_dump) # NOTE: We haven't closed the file opened with fd_state.Open return status