def MakeTestEvaluator(): arena = alloc.SideArena('<MakeTestEvaluator>') mem = state.Mem('', [], {}, arena) exec_opts = state.ExecOpts(mem, None) splitter = legacy.SplitContext(mem) ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter, arena) return ev
def InitEvaluator(): mem = state.Mem('', [], {}, None) state.SetLocalString(mem, 'x', 'xxx') state.SetLocalString(mem, 'y', 'yyy') exec_opts = state.ExecOpts(mem) # Don't need side effects for most things splitter = legacy.SplitContext(mem) return word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
def __init__(self, mem, fd_state, status_lines, funcs, readline, completion, comp_lookup, exec_opts, arena): """ Args: mem: Mem instance for storing variables fd_state: FdState() for managing descriptors status_lines: shared with completion. TODO: Move this to the end. funcs: registry of functions (these names are completed) completion: completion module, if available comp_lookup: completion pattern/action exec_opts: ExecOpts arena: for printing error locations """ self.mem = mem self.fd_state = fd_state self.status_lines = status_lines # function space is different than var space. Not hierarchical. self.funcs = funcs self.completion = completion # Completion hooks, set by 'complete' builtin. self.comp_lookup = comp_lookup # This is for shopt and set -o. They are initialized by flags. self.exec_opts = exec_opts self.exec_opts.readline = readline self.arena = arena self.splitter = legacy.SplitContext(self.mem) self.word_ev = word_eval.NormalWordEvaluator( mem, exec_opts, self.splitter, self) self.arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, self.word_ev) self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.word_ev) self.traps = {} # signal/hook name -> callable self.nodes_to_run = [] # list of nodes, appended to by signal handlers self.dir_stack = state.DirStack() # TODO: Pass these in from main() self.aliases = {} # alias name -> string self.targets = [] # make syntax enters stuff here -- Target() # metaprogramming or regular target syntax # Whether argv[0] is make determines if it is executed self.waiter = process.Waiter() # sleep 5 & puts a (PID, job#) entry here. And then "jobs" displays it. self.job_state = process.JobState() self.loop_level = 0 # for detecting bad top-level break/continue self.tracer = Tracer(exec_opts, mem, self.word_ev) self.check_command_sub_status = False # a hack
def __init__(self, mem, fd_state, funcs, comp_lookup, exec_opts, parse_ctx, devtools): """ Args: mem: Mem instance for storing variables fd_state: FdState() for managing descriptors funcs: dict of functions comp_lookup: registry of completion hooks exec_opts: ExecOpts parse_ctx: for instantiating parsers """ self.mem = mem self.fd_state = fd_state self.funcs = funcs # Completion hooks, set by 'complete' builtin. self.comp_lookup = comp_lookup # This is for shopt and set -o. They are initialized by flags. self.exec_opts = exec_opts self.parse_ctx = parse_ctx self.arena = parse_ctx.arena self.aliases = parse_ctx.aliases # alias name -> string self.dumper = devtools.dumper self.debug_f = devtools.debug_f # Used by ShellFuncAction too self.splitter = legacy.SplitContext(self.mem) self.word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, self.splitter, self.arena, self) self.arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, self.word_ev, self.arena) self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.word_ev, self.arena) self.traps = {} # signal/hook name -> callable self.nodes_to_run = [] # list of nodes, appended to by signal handlers self.dir_stack = state.DirStack() self.targets = [] # make syntax enters stuff here -- Target() # metaprogramming or regular target syntax # Whether argv[0] is make determines if it is executed self.waiter = process.Waiter() # sleep 5 & puts a (PID, job#) entry here. And then "jobs" displays it. self.job_state = process.JobState() self.tracer = Tracer(parse_ctx, exec_opts, mem, self.word_ev, devtools.trace_f) self.loop_level = 0 # for detecting bad top-level break/continue self.check_command_sub_status = False # a hack
def ParseAndEval(code_str): arena = test_lib.MakeArena('<arith_parse_test.py>') parse_ctx = parse_lib.ParseContext(arena, {}) w_parser, _ = parse_ctx.MakeParserForCompletion(code_str, arena) w_parser._Next(lex_mode_e.ARITH) # Calling private method anode = w_parser._ReadArithExpr() # need the right lex state? print('node:', anode) mem = state.Mem('', [], {}, arena) exec_opts = state.ExecOpts(mem, None) splitter = legacy.SplitContext(mem) ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter, arena) arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, ev, arena) value = arith_ev.Eval(anode) return value
def ParseAndEval(code_str): arena = test_lib.MakeArena('<arith_parse_test.py>') w_parser, _ = parse_lib.MakeParserForCompletion(code_str, arena) anode = w_parser._ReadArithExpr() # need the right lex state? if not anode: raise ExprSyntaxError("failed %s" % w_parser.Error()) print('node:', anode) mem = state.Mem('', [], {}, None) exec_opts = state.ExecOpts(mem) splitter = legacy.SplitContext(mem) ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter) arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, ev) value = arith_ev.Eval(anode) return value
def OshMain(argv0, argv, login_shell): spec = args.FlagsAndOptions() spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string spec.ShortFlag('-i') # interactive # TODO: -h too spec.LongFlag('--help') spec.LongFlag('--version') spec.LongFlag( '--ast-format', ['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap', 'none'], default='abbrev-text') spec.LongFlag('--show-ast') # execute and show spec.LongFlag('--fix') spec.LongFlag('--debug-spans') # For oshc translate spec.LongFlag('--print-status') spec.LongFlag( '--trace', ['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now spec.LongFlag('--hijack-shebang') # For benchmarks/*.sh spec.LongFlag('--parser-mem-dump', args.Str) spec.LongFlag('--runtime-mem-dump', args.Str) builtin.AddOptionsToArgSpec(spec) try: opts, opt_index = spec.Parse(argv) except args.UsageError as e: util.usage(str(e)) return 2 if opts.help: loader = util.GetResourceLoader() builtin.Help(['osh-usage'], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 trace_state = util.TraceState() if 'cmd-parse' == opts.trace: util.WrapMethods(cmd_parse.CommandParser, trace_state) if 'word-parse' == opts.trace: util.WrapMethods(word_parse.WordParser, trace_state) if 'lexer' == opts.trace: util.WrapMethods(lexer.Lexer, trace_state) if opt_index == len(argv): dollar0 = argv0 else: dollar0 = argv[opt_index] # the script name, or the arg after -c # TODO: Create a --parse action or 'osh parse' or 'oil osh-parse' # osh-fix # It uses a different memory-management model. It's a batch program and not # an interactive program. pool = alloc.Pool() arena = pool.NewArena() # TODO: Maybe wrap this initialization sequence up in an oil_State, like # lua_State. status_lines = ui.MakeStatusLines() mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ, arena) funcs = {} # Passed to Executor for 'complete', and passed to completion.Init if completion: comp_lookup = completion.CompletionLookup() else: # TODO: NullLookup? comp_lookup = None exec_opts = state.ExecOpts(mem) builtin.SetExecOpts(exec_opts, opts.opt_changes) fd_state = process.FdState() ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, readline, completion, comp_lookup, exec_opts, arena) # NOTE: The rc file can contain both commands and functions... ideally we # would only want to save nodes/lines for the functions. try: rc_path = 'oilrc' arena.PushSource(rc_path) with open(rc_path) as f: rc_line_reader = reader.FileLineReader(f, arena) _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena) try: rc_node = rc_c_parser.ParseWholeFile() if not rc_node: err = rc_c_parser.Error() ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 finally: arena.PopSource() status = ex.Execute(rc_node) #print('oilrc:', status, cflow, file=sys.stderr) # Ignore bad status? except IOError as e: if e.errno != errno.ENOENT: raise if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena) if opts.i: # -c and -i can be combined exec_opts.interactive = True elif opts.i: # force interactive arena.PushSource('<stdin -i>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena) exec_opts.interactive = True else: try: script_name = argv[opt_index] except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') line_reader = reader.InteractiveLineReader(OSH_PS1, arena) exec_opts.interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) else: arena.PushSource(script_name) try: f = fd_state.Open(script_name) except OSError as e: util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena) # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. w_parser, c_parser = parse_lib.MakeParser(line_reader, arena) if exec_opts.interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if HAVE_READLINE: splitter = legacy.SplitContext(mem) ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter) status_out = completion.StatusOutput(status_lines, exec_opts) completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup, status_out, ev) return InteractiveLoop(opts, ex, c_parser, w_parser, line_reader) else: # Parse the whole thing up front #print('Parsing file') _tlog('ParseWholeFile') # TODO: Do I need ParseAndEvalLoop? How is it different than # InteractiveLoop? try: node = c_parser.ParseWholeFile() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stderr) print('parse error: %s' % e.UserErrorString(), file=sys.stderr) return 2 else: # TODO: Remove this older form of error handling. if not node: err = c_parser.Error() assert err, err # can't be empty ui.PrintErrorStack(err, arena, sys.stderr) return 2 # parse error is code 2 do_exec = True if opts.fix: #log('SPANS: %s', arena.spans) osh2oil.PrintAsOil(arena, node, opts.debug_spans) do_exec = False if exec_opts.noexec: do_exec = False # Do this after parsing the entire file. There could be another option to # do it before exiting runtime? if opts.parser_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % os.getpid() with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--parser-mem-dump)', input_path, opts.parser_mem_dump) # -n prints AST, --show-ast prints and executes if exec_opts.noexec or opts.show_ast: if opts.ast_format == 'none': print('AST not printed.', file=sys.stderr) elif opts.ast_format == 'oheap': # TODO: Make this a separate flag? if sys.stdout.isatty(): raise RuntimeError( 'ERROR: Not dumping binary data to a TTY.') f = sys.stdout enc = encode.Params() out = encode.BinOutput(f) encode.EncodeRoot(node, enc, out) else: # text output f = sys.stdout if opts.ast_format in ('text', 'abbrev-text'): ast_f = fmt.DetectConsoleOutput(f) elif opts.ast_format in ('html', 'abbrev-html'): ast_f = fmt.HtmlOutput(f) else: raise AssertionError abbrev_hook = (ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None) tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook) ast_f.FileHeader() fmt.PrintTree(tree, ast_f) ast_f.FileFooter() ast_f.write('\n') #util.log("Execution skipped because 'noexec' is on ") status = 0 if do_exec: _tlog('Execute(node)') status = ex.ExecuteAndRunExitTrap(node) # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub # don't because they call sys.exit(). if opts.runtime_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % os.getpid() with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--runtime-mem-dump)', input_path, opts.runtime_mem_dump) else: status = 0 return status
def _MakeTestEvaluator(): mem = state.Mem('', [], {}, None) exec_opts = state.ExecOpts(mem) splitter = legacy.SplitContext(mem) ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter) return ev
def OshMain(argv0, argv, login_shell): arg_r = args.Reader(argv) try: opts = OSH_SPEC.Parse(arg_r) except args.UsageError as e: ui.usage('osh usage error: %s', e) return 2 if opts.help: loader = util.GetResourceLoader() builtin.Help(['osh-usage'], loader) return 0 if opts.version: # OSH version is the only binary in Oil right now, so it's all one version. _ShowVersion() return 0 # TODO: This should be in interactive mode only? builtin.RegisterSigIntHandler() if arg_r.AtEnd(): dollar0 = argv0 has_main = False else: dollar0 = arg_r.Peek() # the script name, or the arg after -c has_main = True pool = alloc.Pool() arena = pool.NewArena() # NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a # required arg. mem = state.Mem(dollar0, argv[arg_r.i + 1:], posix.environ, arena, has_main=has_main) funcs = {} comp_lookup = completion.CompletionLookup() fd_state = process.FdState() exec_opts = state.ExecOpts(mem, readline) builtin.SetExecOpts(exec_opts, opts.opt_changes) aliases = {} # feedback between runtime and parser parse_ctx = parse_lib.ParseContext(arena, aliases) if opts.debug_file: debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w')) else: debug_f = util.NullDebugFile() debug_f.log('Debug file is %s', opts.debug_file) # Controlled by env variable, flag, or hook? dumper = dev.CrashDumper(posix.environ.get('OSH_CRASH_DUMP_DIR', '')) if opts.xtrace_to_debug_file: trace_f = debug_f else: trace_f = util.DebugFile(sys.stderr) devtools = dev.DevTools(dumper, debug_f, trace_f) ex = cmd_exec.Executor(mem, fd_state, funcs, comp_lookup, exec_opts, parse_ctx, devtools) # NOTE: The rc file can contain both commands and functions... ideally we # would only want to save nodes/lines for the functions. try: rc_path = 'oilrc' arena.PushSource(rc_path) with open(rc_path) as f: rc_line_reader = reader.FileLineReader(f, arena) _, rc_c_parser = parse_ctx.MakeParser(rc_line_reader) try: status = main_loop.Batch(ex, rc_c_parser, arena) finally: arena.PopSource() except IOError as e: if e.errno != errno.ENOENT: raise # Needed in non-interactive shells for @P prompt = ui.Prompt(arena, parse_ctx, ex) ui.PROMPT = prompt if opts.c is not None: arena.PushSource('<command string>') line_reader = reader.StringLineReader(opts.c, arena) if opts.i: # -c and -i can be combined exec_opts.interactive = True elif opts.i: # force interactive arena.PushSource('<stdin -i>') line_reader = reader.InteractiveLineReader(arena, prompt) exec_opts.interactive = True else: try: script_name = arg_r.Peek() except IndexError: if sys.stdin.isatty(): arena.PushSource('<interactive>') line_reader = reader.InteractiveLineReader(arena, prompt) exec_opts.interactive = True else: arena.PushSource('<stdin>') line_reader = reader.FileLineReader(sys.stdin, arena) else: arena.PushSource(script_name) try: f = fd_state.Open(script_name) except OSError as e: util.error("Couldn't open %r: %s", script_name, posix.strerror(e.errno)) return 1 line_reader = reader.FileLineReader(f, arena) # TODO: assert arena.NumSourcePaths() == 1 # TODO: .rc file needs its own arena. w_parser, c_parser = parse_ctx.MakeParser(line_reader) if exec_opts.interactive: # NOTE: We're using a different evaluator here. The completion system can # also run functions... it gets the Executor through Executor._Complete. if HAVE_READLINE: splitter = legacy.SplitContext(mem) # TODO: share with executor. ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter, arena) progress_f = ui.StatusLine() var_action = completion.VariablesActionInternal(ex.mem) root_comp = completion.RootCompleter(ev, comp_lookup, var_action, parse_ctx, progress_f, debug_f) completion.Init(readline, root_comp, debug_f) _InitDefaultCompletions(ex, comp_lookup) return main_loop.Interactive(opts, ex, c_parser, arena) # TODO: Remove this after removing it from benchmarks/osh-runtime. It's no # longer relevant with main_loop. if opts.parser_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--parser-mem-dump)', input_path, opts.parser_mem_dump) nodes_out = [] if exec_opts.noexec else None _tlog('Execute(node)') status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out) if nodes_out is not None: ui.PrintAst(nodes_out, opts) # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub # don't because they call sys.exit(). if opts.runtime_mem_dump: # This might be superstition, but we want to let the value stabilize # after parsing. bash -c 'cat /proc/$$/status' gives different results # with a sleep. time.sleep(0.001) input_path = '/proc/%d/status' % posix.getpid() with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2: contents = f.read() f2.write(contents) log('Wrote %s to %s (--runtime-mem-dump)', input_path, opts.runtime_mem_dump) # NOTE: We haven't closed the file opened with fd_state.Open return status