Beispiel #1
0
def InitExecutor(comp_state=None, arena=None, mem=None):
  arena = arena or MakeArena('<InitExecutor>')

  mem = mem or state.Mem('', [], {}, arena)
  fd_state = process.FdState()
  funcs = {}

  comp_state = comp_state or completion.State()
  readline = None  # simulate not having it
  builtins = {  # Lookup
      builtin_e.HISTORY: builtin.History(readline),

      builtin_e.COMPOPT: builtin_comp.CompOpt(comp_state),
      builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
  }

  # For the tests, we do not use 'readline'.
  exec_opts = state.ExecOpts(mem, None)
  parse_ctx = parse_lib.ParseContext(arena, {})

  debug_f = util.DebugFile(sys.stderr)
  devtools = dev.DevTools(dev.CrashDumper(''), debug_f, debug_f)

  ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
                         parse_ctx, devtools)

  # Add some builtins that depend on the executor!
  complete_builtin = builtin_comp.Complete(ex, comp_state)  # used later
  builtins[builtin_e.COMPLETE] = complete_builtin
  builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(ex)

  return ex
Beispiel #2
0
  def testPipeline2(self):
    cmd_ev = test_lib.InitCommandEvaluator(arena=_ARENA, ext_prog=_EXT_PROG)

    Banner('ls | cut -d . -f 1 | head')
    p = process.Pipeline()
    p.Add(_ExtProc(['ls']))
    p.Add(_ExtProc(['cut', '-d', '.', '-f', '1']))

    node = _CommandNode('head', _ARENA)
    p.AddLast((cmd_ev, node))

    fd_state = process.FdState(_ERRFMT, _JOB_STATE)
    print(p.Run(_WAITER, _FD_STATE))

    # Simulating subshell for each command
    node1 = _CommandNode('ls', _ARENA)
    node2 = _CommandNode('head', _ARENA)
    node3 = _CommandNode('sort --reverse', _ARENA)

    p = process.Pipeline()
    p.Add(Process(process.SubProgramThunk(cmd_ev, node1), _JOB_STATE))
    p.Add(Process(process.SubProgramThunk(cmd_ev, node2), _JOB_STATE))
    p.Add(Process(process.SubProgramThunk(cmd_ev, node3), _JOB_STATE))

    last_thunk = (cmd_ev, _CommandNode('cat', _ARENA))
    p.AddLast(last_thunk)

    print(p.Run(_WAITER, _FD_STATE))
Beispiel #3
0
  def testStdinRedirect(self):
    waiter = process.Waiter()
    fd_state = process.FdState()

    PATH = '_tmp/one-two.txt'
    # Write two lines
    with open(PATH, 'w') as f:
      f.write('one\ntwo\n')

    # Should get the first line twice, because Pop() closes it!

    r = runtime.PathRedirect(Id.Redir_Less, 0, PATH)
    fd_state.Push([r], waiter)
    #line1 = sys.stdin.readline()
    line1 = builtin.ReadLineFromStdin()
    fd_state.Pop()

    fd_state.Push([r], waiter)
    #line2 = sys.stdin.readline()
    line2 = builtin.ReadLineFromStdin()
    fd_state.Pop()

    # sys.stdin.readline() would erroneously return 'two' because of buffering.
    self.assertEqual('one\n', line1)
    self.assertEqual('one\n', line2)
Beispiel #4
0
  def testStdinRedirect(self):
    waiter = process.Waiter(_JOB_STATE, _EXEC_OPTS)
    fd_state = process.FdState(_ERRFMT, _JOB_STATE)

    PATH = '_tmp/one-two.txt'
    # Write two lines
    with open(PATH, 'w') as f:
      f.write('one\ntwo\n')

    # Should get the first line twice, because Pop() closes it!

    r = redirect(Id.Redir_Less, runtime.NO_SPID, redir_loc.Fd(0),
                 redirect_arg.Path(PATH))

    fd_state.Push([r], waiter)
    line1, _ = builtin_misc.ReadLineFromStdin('\n')
    fd_state.Pop()

    fd_state.Push([r], waiter)
    line2, _ = builtin_misc.ReadLineFromStdin('\n')
    fd_state.Pop()

    # sys.stdin.readline() would erroneously return 'two' because of buffering.
    self.assertEqual('one', line1)
    self.assertEqual('one', line2)
Beispiel #5
0
def InitExecutor(parse_ctx=None, comp_lookup=None, arena=None, mem=None):
    if parse_ctx:
        arena = parse_ctx.arena
    else:
        arena or MakeArena('<InitExecutor>')
        parse_ctx = parse_lib.ParseContext(arena, {})

    mem = mem or state.Mem('', [], {}, arena)
    fd_state = process.FdState()
    funcs = {}

    comp_state = completion.State()
    comp_lookup = comp_lookup or completion.Lookup()

    readline = None  # simulate not having it
    builtins = {  # Lookup
        builtin_e.HISTORY: builtin.History(readline),

        builtin_e.COMPOPT: builtin_comp.CompOpt(comp_state),
        builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
    }

    # For the tests, we do not use 'readline'.
    exec_opts = state.ExecOpts(mem, None)

    debug_f = util.DebugFile(sys.stderr)
    exec_deps = cmd_exec.Deps()
    exec_deps.dumper = dev.CrashDumper('')
    exec_deps.debug_f = debug_f
    exec_deps.trace_f = debug_f

    splitter = split.SplitContext(mem)
    exec_deps.splitter = splitter

    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
    exec_deps.word_ev = word_ev

    arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.arith_ev = arith_ev

    word_ev.arith_ev = arith_ev  # Circular

    bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.bool_ev = bool_ev

    tracer = cmd_exec.Tracer(parse_ctx, exec_opts, mem, word_ev, debug_f)
    exec_deps.tracer = tracer

    ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
                           parse_ctx, exec_deps)

    spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
                                            comp_lookup)
    # Add some builtins that depend on the executor!
    complete_builtin = builtin_comp.Complete(spec_builder,
                                             comp_lookup)  # used later
    builtins[builtin_e.COMPLETE] = complete_builtin
    builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)

    return ex
Beispiel #6
0
    def testPipeline2(self):
        arena = test_lib.MakeArena('testPipeline')
        ex = test_lib.InitExecutor(arena=arena)

        Banner('ls | cut -d . -f 1 | head')
        p = process.Pipeline()
        p.Add(_ExtProc(['ls']))
        p.Add(_ExtProc(['cut', '-d', '.', '-f', '1']))

        node = _CommandNode('head', arena)
        p.AddLast((ex, node))

        fd_state = process.FdState()
        print(p.Run(_WAITER, _FD_STATE))

        # Simulating subshell for each command
        node1 = _CommandNode('ls', arena)
        node2 = _CommandNode('head', arena)
        node3 = _CommandNode('sort --reverse', arena)

        p = process.Pipeline()
        p.Add(Process(process.SubProgramThunk(ex, node1)))
        p.Add(Process(process.SubProgramThunk(ex, node2)))
        p.Add(Process(process.SubProgramThunk(ex, node3)))

        last_thunk = (ex, _CommandNode('cat', arena))
        p.AddLast(last_thunk)

        print(p.Run(_WAITER, _FD_STATE))
Beispiel #7
0
    def testOpen(self):
        fd_state = process.FdState(_ERRFMT, _JOB_STATE)

        # This function used to raise BOTH OSError and IOError because Python 2 is
        # inconsistent.
        # We follow Python 3 in preferring OSError.
        # https://stackoverflow.com/questions/29347790/difference-between-ioerror-and-oserror
        self.assertRaises(OSError, fd_state.Open, '_nonexistent_')
        self.assertRaises(OSError, fd_state.Open, 'metrics/')
Beispiel #8
0
  def testHereRedirects(self):
    # NOTE: THis starts another process, which confuses unit test framework!
    return
    fd_state = process.FdState()
    r = HereDocRedirect(Id.Redir_DLess, 0, 'hello\n')
    r.ApplyInParent(fd_state)

    in_str = sys.stdin.readline()
    print(repr(in_str))

    fd_state.PopAndRestore()
Beispiel #9
0
def InitExecutor(arena=None):
    if not arena:
        arena = test_lib.MakeArena('<InitExecutor>')

    mem = state.Mem('', [], {}, None)
    fd_state = process.FdState()
    status_lines = None  # not needed for what we're testing
    funcs = {}
    comp_funcs = {}
    exec_opts = state.ExecOpts(mem)
    return cmd_exec.Executor(mem, fd_state, status_lines, funcs, completion,
                             comp_funcs, exec_opts, arena)
Beispiel #10
0
  def testFilenameRedirect(self):
    print('BEFORE', os.listdir('/dev/fd'))

    fd_state = process.FdState()
    r = process.DescriptorRedirect(Id.Redir_GreatAnd, 1, 2)  # 1>&2
    r.ApplyInParent(fd_state)

    sys.stdout.write('write stdout to stderr\n')
    #os.write(sys.stdout.fileno(), 'write stdout to stderr\n')
    sys.stdout.flush()  # flush required

    fd_state.PopAndRestore()

    fd_state.PushFrame()

    sys.stdout.write('after restoring stdout\n')
    sys.stdout.flush()  # flush required

    r1 = process.FilenameRedirect(Id.Redir_Great, 1, '_tmp/desc3-out.txt')
    r2 = process.FilenameRedirect(Id.Redir_Great, 2, '_tmp/desc3-err.txt')

    r1.ApplyInParent(fd_state)
    r2.ApplyInParent(fd_state)

    sys.stdout.write('stdout to file\n')
    sys.stdout.flush()  # flush required
    sys.stderr.write('stderr to file\n')
    sys.stderr.flush()  # flush required

    fd_state.PopAndRestore()

    r1 = process.FilenameRedirect(Id.Redir_Great, 1, '_tmp/ls-out.txt')
    r2 = process.FilenameRedirect(Id.Redir_Great, 2, '_tmp/ls-err.txt')

    p = Process(
        process.ExternalThunk(['ls', '/error', '.']), fd_state=fd_state,
        redirects=[r1, r2])

    # Bad File Descriptor
    ok = fd_state.SaveAndDup(5, 1)  # 1>&5

    if ok:
      sys.stdout.write('write stdout to stderr\n')
      sys.stdout.flush()  # flush required
      fd_state.PopAndRestore()
    else:
      print('SaveAndDup FAILED')

    print('FDs AFTER', os.listdir('/dev/fd'))
Beispiel #11
0
def InitExecutor(arena=None):
  arena = arena or MakeArena('<InitExecutor>')

  mem = state.Mem('', [], {}, arena)
  fd_state = process.FdState()
  funcs = {}
  comp_funcs = {}
  # For the tests, we do not use 'readline'.
  exec_opts = state.ExecOpts(mem, None)
  parse_ctx = parse_lib.ParseContext(arena, {})

  debug_f = util.DebugFile(sys.stderr)
  devtools = dev.DevTools(dev.CrashDumper(''), debug_f, debug_f)

  return cmd_exec.Executor(mem, fd_state, funcs, comp_funcs, exec_opts,
                           parse_ctx, devtools)
Beispiel #12
0
  def __init__(self, mem, builtins, funcs, completion, comp_lookup, exec_opts,
               make_parser, arena):
    """
    Args:
      mem: Mem instance for storing variables
      builtins: builtin metadata/implementation
      funcs: registry of functions (these names are completed)
      completion: completion module, if available
      comp_lookup: completion pattern/action
      make_parser: Callback for creating a new command parser (eval and source)
      arena: for printing error locations
    """
    self.mem = mem
    self.builtins = builtins
    # function space is different than var space.  Not hierarchical.
    self.funcs = funcs
    self.completion = completion
    # Completion hooks, set by 'complete' builtin.
    self.comp_lookup = comp_lookup
    # This is for shopt and set -o.  They are initialized by flags.
    self.exec_opts = exec_opts
    self.make_parser = make_parser
    self.arena = arena

    self.ev = word_eval.NormalWordEvaluator(mem, exec_opts, self)

    self.mem.last_status = 0  # For $?

    self.traps = {}
    self.fd_state = process.FdState()

    # TODO: Pass these in from main()
    self.aliases = {}  # alias name -> string
    self.targets = []  # make syntax enters stuff here -- Target()
                       # metaprogramming or regular target syntax
                       # Whether argv[0] is make determines if it is executed

    # sleep 5 & puts a (PID, job#) entry here.  And then "jobs" displays it.
    self.jobs = {}

    self.dir_stack = DirStack()

    self.traceback = None
    self.traceback_msg = ''
    self.error_stack = []
Beispiel #13
0
    def setUp(self):
        self.arena = test_lib.MakeArena('process_test.py')

        mem = state.Mem('', [], self.arena, [])
        parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
        mem.exec_opts = exec_opts

        state.InitMem(mem, {}, '0.1')

        self.job_state = process.JobState()
        sig_state = pyos.SignalState()
        self.tracer = dev.Tracer(None, exec_opts, mutable_opts, mem,
                                 mylib.Stderr())
        self.waiter = process.Waiter(self.job_state, exec_opts, sig_state,
                                     self.tracer)
        errfmt = ui.ErrorFormatter(self.arena)
        self.fd_state = process.FdState(errfmt, self.job_state, None,
                                        self.tracer)
        self.ext_prog = process.ExternalProgram('', self.fd_state, errfmt,
                                                util.NullDebugFile())
Beispiel #14
0
  def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
               exec_opts, arena):
    """
    Args:
      mem: Mem instance for storing variables
      status_lines: shared with completion.  TODO: Move this to the end.
      funcs: registry of functions (these names are completed)
      completion: completion module, if available
      comp_lookup: completion pattern/action
      exec_opts: ExecOpts
      arena: for printing error locations
    """
    self.mem = mem
    self.status_lines = status_lines  
    # function space is different than var space.  Not hierarchical.
    self.funcs = funcs
    self.completion = completion
    # Completion hooks, set by 'complete' builtin.
    self.comp_lookup = comp_lookup
    # This is for shopt and set -o.  They are initialized by flags.
    self.exec_opts = exec_opts
    self.arena = arena

    self.ev = word_eval.NormalWordEvaluator(mem, exec_opts, self)
    self.arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, self.ev)
    self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.ev)

    self.traps = {}
    self.fd_state = process.FdState()
    self.dir_stack = []

    # TODO: Pass these in from main()
    self.aliases = {}  # alias name -> string
    self.targets = []  # make syntax enters stuff here -- Target()
                       # metaprogramming or regular target syntax
                       # Whether argv[0] is make determines if it is executed

    self.waiter = process.Waiter()
    # sleep 5 & puts a (PID, job#) entry here.  And then "jobs" displays it.
    self.job_state = process.JobState()
Beispiel #15
0
def ShellMain(lang, argv0, argv, login_shell):
    """Used by bin/osh and bin/oil.

  Args:
    lang: 'osh' or 'oil'
    argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'.  Like busybox.
    login_shell: Was - on the front?
  """
    # Differences between osh and oil:
    # - --help?  I guess Oil has a SUPERSET of OSH options.
    # - oshrc vs oilrc
    # - the parser and executor
    # - Change the prompt in the interactive shell?

    assert lang in ('osh', 'oil'), lang

    arg_r = args.Reader(argv)
    try:
        opts = OSH_SPEC.Parse(arg_r)
    except args.UsageError as e:
        ui.usage('osh usage error: %s', e)
        return 2

    if opts.help:
        loader = util.GetResourceLoader()
        builtin.Help(['%s-usage' % lang], loader)
        return 0
    if opts.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        _ShowVersion()
        return 0

    # TODO: This should be in interactive mode only?
    builtin.RegisterSigIntHandler()

    if arg_r.AtEnd():
        dollar0 = argv0
        has_main = False
    else:
        dollar0 = arg_r.Peek()  # the script name, or the arg after -c
        has_main = True

    pool = alloc.Pool()
    arena = pool.NewArena()

    # NOTE: has_main is only for ${BASH_SOURCE[@} and family.  Could be a
    # required arg.
    mem = state.Mem(dollar0,
                    argv[arg_r.i + 1:],
                    posix.environ,
                    arena,
                    has_main=has_main)
    funcs = {}

    fd_state = process.FdState()
    exec_opts = state.ExecOpts(mem, readline)
    builtin.SetExecOpts(exec_opts, opts.opt_changes)
    aliases = {}  # feedback between runtime and parser

    parse_ctx = parse_lib.ParseContext(arena, aliases)  # For main_loop

    # Three ParseContext instances SHARE aliases.  TODO: Complete aliases.
    comp_arena = pool.NewArena()
    comp_arena.PushSource('<completion>')
    trail1 = parse_lib.Trail()
    comp_ctx = parse_lib.ParseContext(comp_arena, aliases, trail=trail1)

    hist_arena = pool.NewArena()
    hist_arena.PushSource('<history>')
    trail2 = parse_lib.Trail()
    hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2)

    # Deps helps manages dependencies.  These dependencies are circular:
    # - ex and word_ev, arith_ev -- for command sub, arith sub
    # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
    # - ex and builtins (which execute code, like eval)
    # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
    exec_deps = cmd_exec.Deps()

    if opts.debug_file:
        debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w'))
    else:
        debug_f = util.NullDebugFile()
    exec_deps.debug_f = debug_f

    debug_f.log('Debug file is %s', opts.debug_file)

    splitter = split.SplitContext(mem)
    exec_deps.splitter = splitter

    # Controlled by env variable, flag, or hook?
    exec_deps.dumper = dev.CrashDumper(
        posix.environ.get('OSH_CRASH_DUMP_DIR', ''))

    if opts.xtrace_to_debug_file:
        trace_f = debug_f
    else:
        trace_f = util.DebugFile(sys.stderr)
    exec_deps.trace_f = trace_f

    # TODO: Separate comp_state and comp_lookup.
    comp_state = completion.State()
    comp_lookup = completion.Lookup()

    builtins = {  # Lookup
        builtin_e.HISTORY: builtin.History(readline),

        builtin_e.COMPOPT: builtin_comp.CompOpt(comp_state),
        builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
    }
    ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
                           parse_ctx, exec_deps)
    exec_deps.ex = ex

    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
    exec_deps.word_ev = word_ev

    arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.arith_ev = arith_ev
    word_ev.arith_ev = arith_ev  # Another circular dependency

    bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.bool_ev = bool_ev

    tracer = cmd_exec.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
    exec_deps.tracer = tracer

    # HACK for circular deps
    ex.word_ev = word_ev
    ex.arith_ev = arith_ev
    ex.bool_ev = bool_ev
    ex.tracer = tracer

    spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter)
    # Add some builtins that depend on the executor!
    complete_builtin = builtin_comp.Complete(spec_builder,
                                             comp_lookup)  # used later
    builtins[builtin_e.COMPLETE] = complete_builtin
    builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)

    if lang == 'oil':
        # The Oil executor wraps an OSH executor?  It needs to be able to source
        # it.
        ex = oil_cmd_exec.OilExecutor(ex)

    # PromptEvaluator rendering is needed in non-interactive shells for @P.
    prompt_ev = ui.PromptEvaluator(lang, arena, parse_ctx, ex, mem)
    exec_deps.prompt_ev = prompt_ev
    word_ev.prompt_ev = prompt_ev  # HACK for circular deps

    # History evaluation is a no-op if readline is None.
    hist_ev = reader.HistoryEvaluator(readline, hist_ctx, debug_f)

    # Calculate ~/.config/oil/oshrc or oilrc
    # Use ~/.config/oil to avoid cluttering the user's home directory.  Some
    # users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.

    # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
    home_dir = mem.GetVar('HOME')
    assert home_dir.tag == value_e.Str, home_dir
    rc_path = opts.rcfile or os_path.join(home_dir.s, '.config/oil',
                                          lang + 'rc')

    history_filename = os_path.join(home_dir.s, '.config/oil',
                                    'history_' + lang)

    if opts.c is not None:
        arena.PushSource('<command string>')
        line_reader = reader.StringLineReader(opts.c, arena)
        if opts.i:  # -c and -i can be combined
            exec_opts.interactive = True

    elif opts.i:  # force interactive
        arena.PushSource('<stdin -i>')
        # interactive shell only
        line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev)
        exec_opts.interactive = True

    else:
        try:
            script_name = arg_r.Peek()
        except IndexError:
            if sys.stdin.isatty():
                arena.PushSource('<interactive>')
                # interactive shell only
                line_reader = reader.InteractiveLineReader(
                    arena, prompt_ev, hist_ev)
                exec_opts.interactive = True
            else:
                arena.PushSource('<stdin>')
                line_reader = reader.FileLineReader(sys.stdin, arena)
        else:
            arena.PushSource(script_name)
            try:
                f = fd_state.Open(script_name)
            except OSError as e:
                util.error("Couldn't open %r: %s", script_name,
                           posix.strerror(e.errno))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    if lang == 'osh':
        c_parser = parse_ctx.MakeOshParser(line_reader)
    else:
        c_parser = parse_ctx.MakeOilParser(line_reader)

    if exec_opts.interactive:
        # NOTE: We're using a different evaluator here.  The completion system can
        # also run functions... it gets the Executor through Executor._Complete.
        if readline:
            ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
                                                   arena)
            progress_f = ui.StatusLine()
            root_comp = completion.RootCompleter(ev, mem, comp_lookup,
                                                 comp_state, comp_ctx,
                                                 progress_f, debug_f)
            _InitReadline(readline, history_filename, root_comp, debug_f)
            _InitDefaultCompletions(ex, complete_builtin, comp_lookup)

        # NOTE: Call this AFTER _InitDefaultCompletions.
        SourceStartupFile(rc_path, lang, parse_ctx, ex)

        return main_loop.Interactive(opts, ex, c_parser, arena)

    # TODO: Remove this after removing it from benchmarks/osh-runtime.  It's no
    # longer relevant with main_loop.
    if opts.parser_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % posix.getpid()
        with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
            contents = f.read()
            f2.write(contents)
            log('Wrote %s to %s (--parser-mem-dump)', input_path,
                opts.parser_mem_dump)

    nodes_out = [] if exec_opts.noexec else None

    _tlog('Execute(node)')
    status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)

    # Only print nodes if the whole parse succeeded.
    if nodes_out is not None and status == 0:
        ui.PrintAst(nodes_out, opts)

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if opts.runtime_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % posix.getpid()
        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
            contents = f.read()
            f2.write(contents)
            log('Wrote %s to %s (--runtime-mem-dump)', input_path,
                opts.runtime_mem_dump)

    # NOTE: This doesn't cause any spec tests to fail, but it could.
    if posix.environ.get('ASDL_TYPE_CHECK'):
        log('NOTE: Performed %d ASDL_TYPE_CHECKs.', runtime.NUM_TYPE_CHECKS)

    # NOTE: We haven't closed the file opened with fd_state.Open
    return status
Beispiel #16
0
def InitCommandEvaluator(parse_ctx=None, comp_lookup=None, arena=None, mem=None,
                 aliases=None, ext_prog=None):
  opt0_array = state.InitOpts()
  opt_stacks = [None] * option_i.ARRAY_SIZE
  if parse_ctx:
    arena = parse_ctx.arena
    parse_opts = parse_ctx.parse_opts
  else:
    parse_ctx = InitParseContext()

  mem = mem or state.Mem('', [], arena, [])
  exec_opts = optview.Exec(opt0_array, opt_stacks)
  mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
  mem.exec_opts = exec_opts
  state.InitMem(mem, {}, '0.1')
  mutable_opts.Init()

  # No 'readline' in the tests.

  errfmt = ui.ErrorFormatter(arena)
  job_state = process.JobState()
  fd_state = process.FdState(errfmt, job_state)
  aliases = {} if aliases is None else aliases
  procs = {}

  compopt_state = completion.OptionState()
  comp_lookup = comp_lookup or completion.Lookup()

  readline = None  # simulate not having it

  new_var = builtin_assign.NewVar(mem, procs, errfmt)
  assign_builtins = {
      builtin_i.declare: new_var,
      builtin_i.typeset: new_var,
      builtin_i.local: new_var,

      builtin_i.export_: builtin_assign.Export(mem, errfmt),
      builtin_i.readonly: builtin_assign.Readonly(mem, errfmt),
  }
  builtins = {  # Lookup
      builtin_i.echo: builtin_pure.Echo(exec_opts),
      builtin_i.shift: builtin_assign.Shift(mem),

      builtin_i.history: builtin_lib.History(readline, mylib.Stdout()),

      builtin_i.compopt: builtin_comp.CompOpt(compopt_state, errfmt),
      builtin_i.compadjust: builtin_comp.CompAdjust(mem),

      builtin_i.alias: builtin_pure.Alias(aliases, errfmt),
      builtin_i.unalias: builtin_pure.UnAlias(aliases, errfmt),
  }

  debug_f = util.DebugFile(sys.stderr)
  cmd_deps = cmd_eval.Deps()
  cmd_deps.mutable_opts = mutable_opts
  cmd_deps.trap_nodes = []

  search_path = state.SearchPath(mem)
  waiter = process.Waiter(job_state, exec_opts)

  ext_prog = \
      ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)

  cmd_deps.dumper = dev.CrashDumper('')
  cmd_deps.debug_f = debug_f

  splitter = split.SplitContext(mem)

  arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, parse_ctx, errfmt)
  bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, parse_ctx, errfmt)
  expr_ev = expr_eval.OilEvaluator(mem, procs, splitter, errfmt)
  word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
                                          splitter, errfmt)
  cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
                                     assign_builtins, arena, cmd_deps)

  shell_ex = executor.ShellExecutor(
      mem, exec_opts, mutable_opts, procs, builtins, search_path,
      ext_prog, waiter, job_state, fd_state, errfmt)

  assert cmd_ev.mutable_opts is not None, cmd_ev
  prompt_ev = prompt.Evaluator('osh', parse_ctx, mem)

  tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
                      debug_f)

  vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
                      prompt_ev, tracer)

  spec_builder = builtin_comp.SpecBuilder(cmd_ev, parse_ctx, word_ev, splitter,
                                          comp_lookup)
  # Add some builtins that depend on the executor!
  complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
  builtins[builtin_i.complete] = complete_builtin
  builtins[builtin_i.compgen] = builtin_comp.CompGen(spec_builder)

  return cmd_ev
Beispiel #17
0
def Main(lang, arg_r, environ, login_shell, loader, line_input):
    # type: (str, args.Reader, Dict[str, str], bool, pyutil._ResourceLoader, Any) -> int
    """The full shell lifecycle.  Used by bin/osh and bin/oil.

  Args:
    lang: 'osh' or 'oil'
    argv0, arg_r: command line arguments
    environ: environment
    login_shell: Was - on the front?
    loader: to get help, version, grammar, etc.
    line_input: optional GNU readline
  """
    # Differences between osh and oil:
    # - --help?  I guess Oil has a SUPERSET of OSH options.
    # - oshrc vs oilrc
    # - shopt -s oil:all
    # - Change the prompt in the interactive shell?

    # osh-pure:
    # - no oil grammar
    # - no expression evaluator
    # - no interactive shell, or line_input
    # - no process.*
    #   process.{ExternalProgram,Waiter,FdState,JobState,SignalState} -- we want
    #   to evaluate config files without any of these
    # Modules not translated yet: completion, comp_ui, builtin_comp, process
    # - word evaluator
    #   - shouldn't glob?  set -o noglob?  or hard failure?
    #   - ~ shouldn't read from the file system
    #     - I guess it can just be the HOME=HOME?
    # Builtin:
    #   shellvm -c 'echo hi'
    #   shellvm <<< 'echo hi'

    argv0 = arg_r.Peek()
    assert argv0 is not None
    arg_r.Next()

    assert lang in ('osh', 'oil'), lang

    try:
        attrs = flag_spec.ParseMore('main', arg_r)
    except error.Usage as e:
        stderr_line('osh usage error: %s', e.msg)
        return 2
    flag = arg_types.main(attrs.attrs)

    arena = alloc.Arena()
    errfmt = ui.ErrorFormatter(arena)

    help_builtin = builtin_misc.Help(loader, errfmt)
    if flag.help:
        help_builtin.Run(MakeBuiltinArgv(['%s-usage' % lang]))
        return 0
    if flag.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        pyutil.ShowAppVersion('Oil', loader)
        return 0

    no_str = None  # type: str

    debug_stack = []  # type: List[state.DebugFrame]
    if arg_r.AtEnd():
        dollar0 = argv0
    else:
        dollar0 = arg_r.Peek()  # the script name, or the arg after -c

        # Copy quirky bash behavior.
        frame0 = state.DebugFrame(dollar0, 'main', no_str, state.LINE_ZERO, 0,
                                  0)
        debug_stack.append(frame0)

    # Copy quirky bash behavior.
    frame1 = state.DebugFrame(no_str, no_str, no_str, runtime.NO_SPID, 0, 0)
    debug_stack.append(frame1)

    script_name = arg_r.Peek()  # type: Optional[str]
    arg_r.Next()
    mem = state.Mem(dollar0, arg_r.Rest(), arena, debug_stack)

    version_str = pyutil.GetVersion(loader)
    state.InitMem(mem, environ, version_str)

    procs = {}  # type: Dict[str, command__ShFunction]

    job_state = process.JobState()
    fd_state = process.FdState(errfmt, job_state, mem)

    opt_hook = state.OptHook()
    parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
    # TODO: only MutableOpts needs mem, so it's not a true circular dep.
    mem.exec_opts = exec_opts  # circular dep

    if attrs.show_options:  # special case: sh -o
        mutable_opts.ShowOptions([])
        return 0

    # Set these BEFORE processing flags, so they can be overridden.
    if lang == 'oil':
        mutable_opts.SetShoptOption('oil:all', True)

    builtin_pure.SetShellOpts(mutable_opts, attrs.opt_changes,
                              attrs.shopt_changes)
    # feedback between runtime and parser
    aliases = {}  # type: Dict[str, str]

    oil_grammar = None  # type: grammar.Grammar
    #oil_grammar = pyutil.LoadOilGrammar(loader)

    if flag.one_pass_parse and not exec_opts.noexec():
        e_usage('--one-pass-parse requires noexec (-n)')
    parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
    parse_ctx.Init_OnePassParse(flag.one_pass_parse)

    # Three ParseContext instances SHARE aliases.
    comp_arena = alloc.Arena()
    comp_arena.PushSource(source.Unused('completion'))
    trail1 = parse_lib.Trail()
    # one_pass_parse needs to be turned on to complete inside backticks.  TODO:
    # fix the issue where ` gets erased because it's not part of
    # set_completer_delims().
    comp_ctx = parse_lib.ParseContext(comp_arena, parse_opts, aliases,
                                      oil_grammar)
    comp_ctx.Init_Trail(trail1)
    comp_ctx.Init_OnePassParse(True)

    hist_arena = alloc.Arena()
    hist_arena.PushSource(source.Unused('history'))
    trail2 = parse_lib.Trail()
    hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
                                      oil_grammar)
    hist_ctx.Init_Trail(trail2)

    # Deps helps manages dependencies.  These dependencies are circular:
    # - cmd_ev and word_ev, arith_ev -- for command sub, arith sub
    # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
    # - cmd_ev and builtins (which execute code, like eval)
    # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
    cmd_deps = cmd_eval.Deps()
    cmd_deps.mutable_opts = mutable_opts

    # TODO: In general, cmd_deps are shared between the mutually recursive
    # evaluators.  Some of the four below are only shared between a builtin and
    # the CommandEvaluator, so we could put them somewhere else.
    cmd_deps.traps = {}
    cmd_deps.trap_nodes = []  # TODO: Clear on fork() to avoid duplicates

    waiter = process.Waiter(job_state, exec_opts)

    my_pid = posix.getpid()

    debug_path = ''
    debug_dir = environ.get('OSH_DEBUG_DIR')
    if flag.debug_file is not None:
        # --debug-file takes precedence over OSH_DEBUG_DIR
        debug_path = flag.debug_file
    elif debug_dir is not None:
        debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)

    if len(debug_path):
        raise NotImplementedError()
    else:
        debug_f = util.NullDebugFile()  # type: util._DebugFile

    cmd_deps.debug_f = debug_f

    # Not using datetime for dependency reasons.  TODO: maybe show the date at
    # the beginning of the log, and then only show time afterward?  To save
    # space, and make space for microseconds.  (datetime supports microseconds
    # but time.strftime doesn't).
    if mylib.PYTHON:
        iso_stamp = time_.strftime("%Y-%m-%d %H:%M:%S")
        debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid,
                    arg_r.argv)
    if len(debug_path):
        debug_f.log('Writing logs to %r', debug_path)

    interp = environ.get('OSH_HIJACK_SHEBANG', '')
    search_path = state.SearchPath(mem)
    ext_prog = process.ExternalProgram(interp, fd_state, errfmt, debug_f)

    splitter = split.SplitContext(mem)

    # This could just be OSH_DEBUG_STREAMS='debug crash' ?  That might be
    # stuffing too much into one, since a .json crash dump isn't a stream.
    crash_dump_dir = environ.get('OSH_CRASH_DUMP_DIR', '')
    cmd_deps.dumper = dev.CrashDumper(crash_dump_dir)

    if flag.xtrace_to_debug_file:
        trace_f = debug_f
    else:
        trace_f = util.DebugFile(mylib.Stderr())

    #comp_lookup = completion.Lookup()

    # Various Global State objects to work around readline interfaces
    #compopt_state = completion.OptionState()
    #comp_ui_state = comp_ui.State()
    #prompt_state = comp_ui.PromptState()

    dir_stack = state.DirStack()

    #
    # Initialize builtins that don't depend on evaluators
    #

    builtins = {}  # type: Dict[int, vm._Builtin]

    AddPure(builtins, mem, procs, mutable_opts, aliases, search_path, errfmt)
    AddIO(builtins, mem, dir_stack, exec_opts, splitter, parse_ctx, errfmt)

    builtins[builtin_i.help] = help_builtin

    #
    # Initialize Evaluators
    #

    arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, parse_ctx, errfmt)
    bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, parse_ctx, errfmt)
    expr_ev = None  # type: expr_eval.OilEvaluator
    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)

    assign_b = InitAssignmentBuiltins(mem, procs, errfmt)
    cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
                                       arena, cmd_deps)

    shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
                                      builtins, search_path, ext_prog, waiter,
                                      job_state, fd_state, errfmt)
    #shell_ex = NullExecutor(exec_opts, mutable_opts, procs, builtins)

    # PromptEvaluator rendering is needed in non-interactive shells for @P.
    prompt_ev = prompt.Evaluator(lang, parse_ctx, mem)
    tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
                        trace_f)

    # Wire up circular dependencies.
    vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
                        prompt_ev, tracer)

    #
    # Initialize builtins that depend on evaluators
    #

    # note: 'printf -v a[i]' and 'unset a[i]' require same deps
    builtins[builtin_i.printf] = builtin_printf.Printf(mem, exec_opts,
                                                       parse_ctx, arith_ev,
                                                       errfmt)

    builtins[builtin_i.unset] = builtin_assign.Unset(mem, exec_opts, procs,
                                                     parse_ctx, arith_ev,
                                                     errfmt)
    builtins[builtin_i.eval] = builtin_meta.Eval(parse_ctx, exec_opts, cmd_ev)

    #source_builtin = builtin_meta.Source(parse_ctx, search_path, cmd_ev,
    #fd_state, errfmt)
    #builtins[builtin_i.source] = source_builtin
    #builtins[builtin_i.dot] = source_builtin

    AddMeta(builtins, shell_ex, mutable_opts, mem, procs, aliases, search_path,
            errfmt)
    AddBlock(builtins, mem, mutable_opts, dir_stack, cmd_ev, errfmt)

    #sig_state = process.SignalState()
    #sig_state.InitShell()

    #builtins[builtin_i.trap] = builtin_process.Trap(sig_state, cmd_deps.traps,
    #                                                cmd_deps.trap_nodes,
    #                                                parse_ctx, errfmt)

    if flag.c is not None:
        arena.PushSource(source.CFlag())
        line_reader = reader.StringLineReader(flag.c,
                                              arena)  # type: reader._Reader
        if flag.i:  # -c and -i can be combined
            mutable_opts.set_interactive()

    elif flag.i:  # force interactive
        raise NotImplementedError()

    else:
        if script_name is None:
            stdin = mylib.Stdin()
            arena.PushSource(source.Stdin(''))
            line_reader = reader.FileLineReader(stdin, arena)
        else:
            arena.PushSource(source.MainFile(script_name))
            try:
                f = fd_state.Open(script_name)
                #f = mylib.open(script_name)
            except OSError as e:
                stderr_line("osh: Couldn't open %r: %s", script_name,
                            pyutil.strerror(e))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    c_parser = parse_ctx.MakeOshParser(line_reader)

    if exec_opts.interactive():
        raise NotImplementedError()

    if exec_opts.noexec():
        status = 0
        try:
            node = main_loop.ParseWholeFile(c_parser)
        except error.Parse as e:
            ui.PrettyPrintError(e, arena)
            status = 2

        if status == 0:
            if flag.parser_mem_dump is not None:  # only valid in -n mode
                input_path = '/proc/%d/status' % posix.getpid()
                pyutil.CopyFile(input_path, flag.parser_mem_dump)

            ui.PrintAst(node, flag)
    else:
        if flag.parser_mem_dump is not None:
            e_usage('--parser-mem-dump can only be used with -n')

        try:
            status = main_loop.Batch(cmd_ev,
                                     c_parser,
                                     arena,
                                     cmd_flags=cmd_eval.IsMainProgram)
        except util.UserExit as e:
            status = e.status
        box = [status]
        cmd_ev.MaybeRunExitTrap(box)
        status = box[0]

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if flag.runtime_mem_dump is not None:
        input_path = '/proc/%d/status' % posix.getpid()
        pyutil.CopyFile(input_path, flag.runtime_mem_dump)

    # NOTE: We haven't closed the file opened with fd_state.Open
    return status
Beispiel #18
0
def ShellMain(lang, argv0, argv, login_shell):
    """Used by bin/osh and bin/oil.

  Args:
    lang: 'osh' or 'oil'
    argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'.  Like busybox.
    login_shell: Was - on the front?
  """
    # Differences between osh and oil:
    # - --help?  I guess Oil has a SUPERSET of OSH options.
    # - oshrc vs oilrc
    # - the parser and executor
    # - Change the prompt in the interactive shell?

    assert lang in ('osh', 'oil'), lang

    arg_r = args.Reader(argv)
    try:
        opts = OSH_SPEC.Parse(arg_r)
    except args.UsageError as e:
        ui.usage('osh usage error: %s', e)
        return 2

    # NOTE: This has a side effect of deleting _OVM_* from the environment!
    # TODO: Thread this throughout the program, and get rid of the global
    # variable in core/util.py.  Rename to InitResourceLaoder().  It's now only
    # used for the 'help' builtin and --version.
    loader = pyutil.GetResourceLoader()

    if opts.help:
        builtin.Help(['%s-usage' % lang], loader)
        return 0
    if opts.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        _ShowVersion()
        return 0

    if arg_r.AtEnd():
        dollar0 = argv0
        has_main = False
    else:
        dollar0 = arg_r.Peek()  # the script name, or the arg after -c
        has_main = True

    arena = alloc.Arena()

    # NOTE: has_main is only for ${BASH_SOURCE[@} and family.  Could be a
    # required arg.
    mem = state.Mem(dollar0,
                    argv[arg_r.i + 1:],
                    posix.environ,
                    arena,
                    has_main=has_main)
    funcs = {}

    fd_state = process.FdState()
    exec_opts = state.ExecOpts(mem, line_input)
    builtin.SetExecOpts(exec_opts, opts.opt_changes)
    aliases = {}  # feedback between runtime and parser

    if opts.one_pass_parse and not exec_opts.noexec:
        raise args.UsageError('--one-pass-parse requires noexec (-n)')
    parse_ctx = parse_lib.ParseContext(arena,
                                       aliases,
                                       one_pass_parse=opts.one_pass_parse)

    # Three ParseContext instances SHARE aliases.
    comp_arena = alloc.Arena()
    comp_arena.PushSource(source.Unused('completion'))
    trail1 = parse_lib.Trail()
    # one_pass_parse needs to be turned on to complete inside backticks.  TODO:
    # fix the issue where ` gets erased because it's not part of
    # set_completer_delims().
    comp_ctx = parse_lib.ParseContext(comp_arena,
                                      aliases,
                                      trail=trail1,
                                      one_pass_parse=True)

    hist_arena = alloc.Arena()
    hist_arena.PushSource(source.Unused('history'))
    trail2 = parse_lib.Trail()
    hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2)

    # Deps helps manages dependencies.  These dependencies are circular:
    # - ex and word_ev, arith_ev -- for command sub, arith sub
    # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
    # - ex and builtins (which execute code, like eval)
    # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
    exec_deps = cmd_exec.Deps()

    my_pid = posix.getpid()

    debug_path = ''
    debug_dir = posix.environ.get('OSH_DEBUG_DIR')
    if opts.debug_file:  # --debug-file takes precedence over OSH_DEBUG_DIR
        debug_path = opts.debug_file
    elif debug_dir:
        debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)

    if debug_path:
        # This will be created as an empty file if it doesn't exist, or it could be
        # a pipe.
        try:
            debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
        except OSError as e:
            util.error("Couldn't open %r: %s", debug_path,
                       posix.strerror(e.errno))
            return 2
    else:
        debug_f = util.NullDebugFile()

    exec_deps.debug_f = debug_f

    # Not using datetime for dependency reasons.  TODO: maybe show the date at
    # the beginning of the log, and then only show time afterward?  To save
    # space, and make space for microseconds.  (datetime supports microseconds
    # but time.strftime doesn't).
    iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
    debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
    if debug_path:
        debug_f.log('Writing logs to %r', debug_path)

    interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
    exec_deps.ext_prog = process.ExternalProgram(interp, fd_state, debug_f)

    splitter = split.SplitContext(mem)
    exec_deps.splitter = splitter

    # This could just be OSH_DEBUG_STREAMS='debug crash' ?  That might be
    # stuffing too much into one, since a .json crash dump isn't a stream.
    crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
    exec_deps.dumper = dev.CrashDumper(crash_dump_dir)

    if opts.xtrace_to_debug_file:
        trace_f = debug_f
    else:
        trace_f = util.DebugFile(sys.stderr)
    exec_deps.trace_f = trace_f

    comp_lookup = completion.Lookup()

    #
    # Various Global State objects to work around readline interfaces
    #

    compopt_state = completion.OptionState()
    comp_ui_state = comp_ui.State()
    prompt_state = comp_ui.PromptState()

    builtins = {  # Lookup
        builtin_e.HISTORY: builtin.History(line_input),

        builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state),
        builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
    }
    ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
                           parse_ctx, exec_deps)
    exec_deps.ex = ex

    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
    exec_deps.word_ev = word_ev

    arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.arith_ev = arith_ev
    word_ev.arith_ev = arith_ev  # Another circular dependency

    bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.bool_ev = bool_ev

    tracer = dev.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
    exec_deps.tracer = tracer

    # HACK for circular deps
    ex.word_ev = word_ev
    ex.arith_ev = arith_ev
    ex.bool_ev = bool_ev
    ex.tracer = tracer

    spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
                                            comp_lookup)

    # Add some builtins that depend on the executor!
    complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
    builtins[builtin_e.COMPLETE] = complete_builtin
    builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)

    if lang == 'oil':
        # The Oil executor wraps an OSH executor?  It needs to be able to source
        # it.
        ex = oil_cmd_exec.OilExecutor(ex)

    # PromptEvaluator rendering is needed in non-interactive shells for @P.
    prompt_ev = prompt.Evaluator(lang, parse_ctx, ex, mem)
    exec_deps.prompt_ev = prompt_ev
    word_ev.prompt_ev = prompt_ev  # HACK for circular deps

    # History evaluation is a no-op if line_input is None.
    hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)

    # Calculate ~/.config/oil/oshrc or oilrc
    # Use ~/.config/oil to avoid cluttering the user's home directory.  Some
    # users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.

    # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
    home_dir = process.GetHomeDir()
    assert home_dir is not None
    rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil', lang + 'rc')

    history_filename = os_path.join(home_dir, '.config/oil', 'history_' + lang)

    if opts.c is not None:
        arena.PushSource(source.CFlag())
        line_reader = reader.StringLineReader(opts.c, arena)
        if opts.i:  # -c and -i can be combined
            exec_opts.interactive = True

    elif opts.i:  # force interactive
        arena.PushSource(source.Stdin(' -i'))
        # interactive shell only
        line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
                                                   line_input, prompt_state)
        exec_opts.interactive = True

    else:
        try:
            script_name = arg_r.Peek()
        except IndexError:
            if sys.stdin.isatty():
                arena.PushSource(source.Interactive())
                # interactive shell only
                line_reader = reader.InteractiveLineReader(
                    arena, prompt_ev, hist_ev, line_input, prompt_state)
                exec_opts.interactive = True
            else:
                arena.PushSource(source.Stdin(''))
                line_reader = reader.FileLineReader(sys.stdin, arena)
        else:
            arena.PushSource(source.MainFile(script_name))
            try:
                f = fd_state.Open(script_name)
            except OSError as e:
                util.error("Couldn't open %r: %s", script_name,
                           posix.strerror(e.errno))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    if lang == 'osh':
        c_parser = parse_ctx.MakeOshParser(line_reader)
    else:
        c_parser = parse_ctx.MakeOilParser(line_reader)

    # NOTE: SIGINT is temporarily enabled during readline() by
    # frontend/reader.py.
    # It's treated differently than SIGQUIT and SIGTSTP because Python handles it
    # with KeyboardInterrupt.  We don't want KeyboardInterrupt at arbitrary
    # points in a non-interactive shell.  (e.g. osh -c 'sleep 5' then Ctrl-C)
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    if exec_opts.interactive:
        if line_input:
            # NOTE: We're using a different WordEvaluator here.
            ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
                                                   arena)
            root_comp = completion.RootCompleter(ev, mem, comp_lookup,
                                                 compopt_state, comp_ui_state,
                                                 comp_ctx, debug_f)

            term_width = 0
            if opts.completion_display == 'nice':
                try:
                    term_width = libc.get_terminal_width()
                except IOError:  # stdin not a terminal
                    pass

            if term_width != 0:
                display = comp_ui.NiceDisplay(term_width, comp_ui_state,
                                              prompt_state, debug_f,
                                              line_input)
            else:
                display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                                 debug_f)

            _InitReadline(line_input, history_filename, root_comp, display,
                          debug_f)
            _InitDefaultCompletions(ex, complete_builtin, comp_lookup)

        else:  # Without readline module
            display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                             debug_f)

        # The shell itself should ignore Ctrl-\.
        signal.signal(signal.SIGQUIT, signal.SIG_IGN)

        # This prevents Ctrl-Z from suspending OSH in interactive mode.  But we're
        # not getting notification via wait() that the child stopped?
        signal.signal(signal.SIGTSTP, signal.SIG_IGN)

        # Register a callback to receive terminal width changes.
        signal.signal(signal.SIGWINCH, lambda x, y: display.OnWindowChange())

        # NOTE: Call this AFTER _InitDefaultCompletions.
        SourceStartupFile(rc_path, lang, parse_ctx, ex)

        line_reader.Reset()  # After sourcing startup file, render $PS1
        return main_loop.Interactive(opts, ex, c_parser, display, arena)

    nodes_out = [] if exec_opts.noexec else None

    if nodes_out is None and opts.parser_mem_dump:
        raise args.UsageError('--parser-mem-dump can only be used with -n')

    _tlog('Execute(node)')
    status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)

    # Only print nodes if the whole parse succeeded.
    if nodes_out is not None and status == 0:
        if opts.parser_mem_dump:  # only valid in -n mode
            # This might be superstition, but we want to let the value stabilize
            # after parsing.  bash -c 'cat /proc/$$/status' gives different results
            # with a sleep.
            time.sleep(0.001)
            input_path = '/proc/%d/status' % posix.getpid()
            with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
                contents = f.read()
                f2.write(contents)
                log('Wrote %s to %s (--parser-mem-dump)', input_path,
                    opts.parser_mem_dump)

        ui.PrintAst(nodes_out, opts)

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if opts.runtime_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % posix.getpid()
        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
            contents = f.read()
            f2.write(contents)
            log('Wrote %s to %s (--runtime-mem-dump)', input_path,
                opts.runtime_mem_dump)

    # NOTE: We haven't closed the file opened with fd_state.Open
    return status
Beispiel #19
0
from core.util import log
from core import test_lib

Process = process.Process
ExternalThunk = process.ExternalThunk


def Banner(msg):
    print('-' * 60)
    print(msg)


_WAITER = process.Waiter()
_ARENA = test_lib.MakeArena('process_test.py')
_ERRFMT = ui.ErrorFormatter(_ARENA)
_FD_STATE = process.FdState(_ERRFMT)
_EXT_PROG = process.ExternalProgram(False, _FD_STATE, _ERRFMT,
                                    util.NullDebugFile())


def _CommandNode(code_str, arena):
    c_parser = test_lib.InitCommandParser(code_str, arena=arena)
    return c_parser.ParseLogicalLine()


def _ExtProc(argv):
    arg_vec = arg_vector(argv, [0] * len(argv))
    return Process(ExternalThunk(_EXT_PROG, arg_vec, {}))


class ProcessTest(unittest.TestCase):
Beispiel #20
0
def ShellMain(lang, argv0, argv, login_shell):
    # type: (str, str, List[str], bool) -> int
    """Used by bin/osh and bin/oil.

  Args:
    lang: 'osh' or 'oil'
    argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'.  Like busybox.
    login_shell: Was - on the front?
  """
    # Differences between osh and oil:
    # - --help?  I guess Oil has a SUPERSET of OSH options.
    # - oshrc vs oilrc
    # - the parser and executor
    # - Change the prompt in the interactive shell?

    assert lang in ('osh', 'oil'), lang

    arg_r = args.Reader(argv)
    try:
        opts = OSH_SPEC.Parse(arg_r)
    except args.UsageError as e:
        ui.Stderr('osh usage error: %s', e.msg)
        return 2

    # NOTE: This has a side effect of deleting _OVM_* from the environment!
    # TODO: Thread this throughout the program, and get rid of the global
    # variable in core/util.py.  Rename to InitResourceLaoder().  It's now only
    # used for the 'help' builtin and --version.
    loader = pyutil.GetResourceLoader()

    if opts.help:
        builtin_misc.Help(['%s-usage' % lang], loader)
        return 0
    if opts.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        _ShowVersion()
        return 0

    no_str = None  # type: str

    debug_stack = []
    if arg_r.AtEnd():
        dollar0 = argv0
    else:
        dollar0 = arg_r.Peek()  # the script name, or the arg after -c

        # Copy quirky bash behavior.
        frame0 = state.DebugFrame(dollar0, 'main', no_str, state.LINE_ZERO, 0,
                                  0)
        debug_stack.append(frame0)

    # Copy quirky bash behavior.
    frame1 = state.DebugFrame(no_str, no_str, no_str, runtime.NO_SPID, 0, 0)
    debug_stack.append(frame1)

    arena = alloc.Arena()
    errfmt = ui.ErrorFormatter(arena)

    mem = state.Mem(dollar0, argv[arg_r.i + 1:], arena, debug_stack)
    state.InitMem(mem, posix.environ)
    builtin_funcs.Init(mem)

    procs = {}

    job_state = process.JobState()
    fd_state = process.FdState(errfmt, job_state)

    opt_hook = ShellOptHook(line_input)
    parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
    # TODO: only MutableOpts needs mem, so it's not a true circular dep.
    mem.exec_opts = exec_opts  # circular dep

    if opts.show_options:  # special case: sh -o
        mutable_opts.ShowOptions([])
        return 0

    # Set these BEFORE processing flags, so they can be overridden.
    if lang == 'oil':
        mutable_opts.SetShoptOption('oil:all', True)

    builtin_pure.SetShellOpts(mutable_opts, opts.opt_changes,
                              opts.shopt_changes)
    aliases = {}  # feedback between runtime and parser

    oil_grammar = meta.LoadOilGrammar(loader)

    if opts.one_pass_parse and not exec_opts.noexec():
        raise args.UsageError('--one-pass-parse requires noexec (-n)')
    parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
    parse_ctx.Init_OnePassParse(opts.one_pass_parse)

    # Three ParseContext instances SHARE aliases.
    comp_arena = alloc.Arena()
    comp_arena.PushSource(source.Unused('completion'))
    trail1 = parse_lib.Trail()
    # one_pass_parse needs to be turned on to complete inside backticks.  TODO:
    # fix the issue where ` gets erased because it's not part of
    # set_completer_delims().
    comp_ctx = parse_lib.ParseContext(comp_arena, parse_opts, aliases,
                                      oil_grammar)
    comp_ctx.Init_Trail(trail1)
    comp_ctx.Init_OnePassParse(True)

    hist_arena = alloc.Arena()
    hist_arena.PushSource(source.Unused('history'))
    trail2 = parse_lib.Trail()
    hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
                                      oil_grammar)
    hist_ctx.Init_Trail(trail2)

    # Deps helps manages dependencies.  These dependencies are circular:
    # - ex and word_ev, arith_ev -- for command sub, arith sub
    # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
    # - ex and builtins (which execute code, like eval)
    # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
    exec_deps = cmd_exec.Deps()
    exec_deps.mutable_opts = mutable_opts

    # TODO: In general, exec_deps are shared between the mutually recursive
    # evaluators.  Some of the four below are only shared between a builtin and
    # the Executor, so we could put them somewhere else.
    exec_deps.traps = {}
    exec_deps.trap_nodes = []  # TODO: Clear on fork() to avoid duplicates

    exec_deps.job_state = job_state
    exec_deps.waiter = process.Waiter(job_state, exec_opts)
    exec_deps.errfmt = errfmt

    my_pid = posix.getpid()

    debug_path = ''
    debug_dir = posix.environ.get('OSH_DEBUG_DIR')
    if opts.debug_file:  # --debug-file takes precedence over OSH_DEBUG_DIR
        debug_path = opts.debug_file
    elif debug_dir:
        debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)

    if debug_path:
        # This will be created as an empty file if it doesn't exist, or it could be
        # a pipe.
        try:
            debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
        except OSError as e:
            ui.Stderr("osh: Couldn't open %r: %s", debug_path,
                      posix.strerror(e.errno))
            return 2
    else:
        debug_f = util.NullDebugFile()

    exec_deps.debug_f = debug_f

    # Not using datetime for dependency reasons.  TODO: maybe show the date at
    # the beginning of the log, and then only show time afterward?  To save
    # space, and make space for microseconds.  (datetime supports microseconds
    # but time.strftime doesn't).
    iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
    debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
    if debug_path:
        debug_f.log('Writing logs to %r', debug_path)

    interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
    exec_deps.search_path = state.SearchPath(mem)
    exec_deps.ext_prog = process.ExternalProgram(interp, fd_state,
                                                 exec_deps.search_path, errfmt,
                                                 debug_f)

    splitter = split.SplitContext(mem)

    # split() builtin
    # TODO: Accept IFS as a named arg?  split('a b', IFS=' ')
    builtin_funcs.SetGlobalFunc(
        mem,
        'split',
        lambda s, ifs=None: splitter.SplitForWordEval(s, ifs=ifs))

    # glob() builtin
    # TODO: This is instantiation is duplicated in osh/word_eval.py
    globber = glob_.Globber(exec_opts)
    builtin_funcs.SetGlobalFunc(mem, 'glob', lambda s: globber.OilFuncCall(s))

    # This could just be OSH_DEBUG_STREAMS='debug crash' ?  That might be
    # stuffing too much into one, since a .json crash dump isn't a stream.
    crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
    exec_deps.dumper = dev.CrashDumper(crash_dump_dir)

    if opts.xtrace_to_debug_file:
        trace_f = debug_f
    else:
        trace_f = util.DebugFile(sys.stderr)
    exec_deps.trace_f = trace_f

    comp_lookup = completion.Lookup()

    # Various Global State objects to work around readline interfaces
    compopt_state = completion.OptionState()
    comp_ui_state = comp_ui.State()
    prompt_state = comp_ui.PromptState()

    dir_stack = state.DirStack()

    new_var = builtin_assign.NewVar(mem, procs, errfmt)

    builtins = {  # Lookup
        builtin_i.echo: builtin_pure.Echo(exec_opts),
        builtin_i.printf: builtin_printf.Printf(mem, parse_ctx, errfmt),

        builtin_i.pushd: builtin_misc.Pushd(mem, dir_stack, errfmt),
        builtin_i.popd: builtin_misc.Popd(mem, dir_stack, errfmt),
        builtin_i.dirs: builtin_misc.Dirs(mem, dir_stack, errfmt),
        builtin_i.pwd: builtin_misc.Pwd(mem, errfmt),

        builtin_i.times: builtin_misc.Times(),
        builtin_i.read: builtin_misc.Read(splitter, mem),
        builtin_i.help: builtin_misc.Help(loader, errfmt),
        builtin_i.history: builtin_misc.History(line_input),

        # Completion (more added below)
        builtin_i.compopt: builtin_comp.CompOpt(compopt_state, errfmt),
        builtin_i.compadjust: builtin_comp.CompAdjust(mem),

        # test / [ differ by need_right_bracket
        builtin_i.test: builtin_bracket.Test(False, exec_opts, errfmt),
        builtin_i.bracket: builtin_bracket.Test(True, exec_opts, errfmt),

        # ShAssignment (which are pure)
        builtin_i.declare: new_var,
        builtin_i.typeset: new_var,
        builtin_i.local: new_var,

        builtin_i.export_: builtin_assign.Export(mem, errfmt),
        builtin_i.readonly: builtin_assign.Readonly(mem, errfmt),

        builtin_i.unset: builtin_assign.Unset(mem, procs, errfmt),
        builtin_i.shift: builtin_assign.Shift(mem),

        # Pure
        builtin_i.set: builtin_pure.Set(mutable_opts, mem),
        builtin_i.shopt: builtin_pure.Shopt(mutable_opts),

        builtin_i.alias: builtin_pure.Alias(aliases, errfmt),
        builtin_i.unalias: builtin_pure.UnAlias(aliases, errfmt),

        builtin_i.type: builtin_pure.Type(procs, aliases, exec_deps.search_path),
        builtin_i.hash: builtin_pure.Hash(exec_deps.search_path),
        builtin_i.getopts: builtin_pure.GetOpts(mem, errfmt),

        builtin_i.colon: builtin_pure.Boolean(0),  # a "special" builtin 
        builtin_i.true_: builtin_pure.Boolean(0),
        builtin_i.false_: builtin_pure.Boolean(1),

        # Process
        builtin_i.wait: builtin_process.Wait(exec_deps.waiter,
                                             exec_deps.job_state, mem, errfmt),
        builtin_i.jobs: builtin_process.Jobs(exec_deps.job_state),
        builtin_i.fg: builtin_process.Fg(exec_deps.job_state, exec_deps.waiter),
        builtin_i.bg: builtin_process.Bg(exec_deps.job_state),
        builtin_i.umask: builtin_process.Umask(),

        # Oil
        builtin_i.push: builtin_oil.Push(mem, errfmt),
        builtin_i.append: builtin_oil.Append(mem, errfmt),

        builtin_i.write: builtin_oil.Write(mem, errfmt),
        builtin_i.getline: builtin_oil.Getline(mem, errfmt),

        builtin_i.repr: builtin_oil.Repr(mem, errfmt),
        builtin_i.use: builtin_oil.Use(mem, errfmt),
        builtin_i.opts: builtin_oil.Opts(mem, errfmt),
    }

    arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, errfmt)
    bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, errfmt)
    expr_ev = expr_eval.OilEvaluator(mem, procs, errfmt)
    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)
    ex = cmd_exec.Executor(mem, fd_state, procs, builtins, exec_opts,
                           parse_ctx, exec_deps)
    # PromptEvaluator rendering is needed in non-interactive shells for @P.
    prompt_ev = prompt.Evaluator(lang, parse_ctx, mem)
    tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
                        trace_f)

    # Wire up circular dependencies.
    vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, ex, prompt_ev,
                        tracer)

    spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
                                            comp_lookup)

    # Add some builtins that depend on the executor!
    complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
    builtins[builtin_i.complete] = complete_builtin
    builtins[builtin_i.compgen] = builtin_comp.CompGen(spec_builder)
    builtins[builtin_i.cd] = builtin_misc.Cd(mem, dir_stack, ex, errfmt)
    builtins[builtin_i.json] = builtin_oil.Json(mem, ex, errfmt)

    sig_state = process.SignalState()
    sig_state.InitShell()

    builtins[builtin_i.trap] = builtin_process.Trap(sig_state, exec_deps.traps,
                                                    exec_deps.trap_nodes, ex,
                                                    errfmt)

    # History evaluation is a no-op if line_input is None.
    hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)

    if opts.c is not None:
        arena.PushSource(source.CFlag())
        line_reader = reader.StringLineReader(opts.c, arena)
        if opts.i:  # -c and -i can be combined
            mutable_opts.set_interactive()

    elif opts.i:  # force interactive
        arena.PushSource(source.Stdin(' -i'))
        line_reader = py_reader.InteractiveLineReader(arena, prompt_ev,
                                                      hist_ev, line_input,
                                                      prompt_state)
        mutable_opts.set_interactive()

    else:
        script_name = arg_r.Peek()
        if script_name is None:
            if sys.stdin.isatty():
                arena.PushSource(source.Interactive())
                line_reader = py_reader.InteractiveLineReader(
                    arena, prompt_ev, hist_ev, line_input, prompt_state)
                mutable_opts.set_interactive()
            else:
                arena.PushSource(source.Stdin(''))
                line_reader = reader.FileLineReader(sys.stdin, arena)
        else:
            arena.PushSource(source.MainFile(script_name))
            try:
                f = fd_state.Open(script_name)
            except OSError as e:
                ui.Stderr("osh: Couldn't open %r: %s", script_name,
                          posix.strerror(e.errno))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    c_parser = parse_ctx.MakeOshParser(line_reader)

    if exec_opts.interactive():
        # Calculate ~/.config/oil/oshrc or oilrc
        # Use ~/.config/oil to avoid cluttering the user's home directory.  Some
        # users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.

        # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
        home_dir = passwd.GetMyHomeDir()
        assert home_dir is not None
        rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil',
                                              lang + 'rc')

        history_filename = os_path.join(home_dir, '.config/oil',
                                        'history_' + lang)

        if line_input:
            # NOTE: We're using a different WordEvaluator here.
            ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
                                                   errfmt)

            ev.arith_ev = arith_ev
            ev.expr_ev = expr_ev
            ev.prompt_ev = prompt_ev
            ev.CheckCircularDeps()

            root_comp = completion.RootCompleter(ev, mem, comp_lookup,
                                                 compopt_state, comp_ui_state,
                                                 comp_ctx, debug_f)

            term_width = 0
            if opts.completion_display == 'nice':
                try:
                    term_width = libc.get_terminal_width()
                except IOError:  # stdin not a terminal
                    pass

            if term_width != 0:
                display = comp_ui.NiceDisplay(term_width, comp_ui_state,
                                              prompt_state, debug_f,
                                              line_input)
            else:
                display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                                 debug_f)

            _InitReadline(line_input, history_filename, root_comp, display,
                          debug_f)
            _InitDefaultCompletions(ex, complete_builtin, comp_lookup)

        else:  # Without readline module
            display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                             debug_f)

        sig_state.InitInteractiveShell(display)

        # NOTE: Call this AFTER _InitDefaultCompletions.
        try:
            SourceStartupFile(rc_path, lang, parse_ctx, ex)
        except util.UserExit as e:
            return e.status

        line_reader.Reset()  # After sourcing startup file, render $PS1

        prompt_plugin = prompt.UserPlugin(mem, parse_ctx, ex)
        try:
            status = main_loop.Interactive(opts, ex, c_parser, display,
                                           prompt_plugin, errfmt)
            if ex.MaybeRunExitTrap():
                status = ex.LastStatus()
        except util.UserExit as e:
            status = e.status
        return status

    nodes_out = [] if exec_opts.noexec() else None

    if nodes_out is None and opts.parser_mem_dump:
        raise args.UsageError('--parser-mem-dump can only be used with -n')

    _tlog('Execute(node)')
    try:
        status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
        if ex.MaybeRunExitTrap():
            status = ex.LastStatus()
    except util.UserExit as e:
        status = e.status

    # Only print nodes if the whole parse succeeded.
    if nodes_out is not None and status == 0:
        if opts.parser_mem_dump:  # only valid in -n mode
            # This might be superstition, but we want to let the value stabilize
            # after parsing.  bash -c 'cat /proc/$$/status' gives different results
            # with a sleep.
            time.sleep(0.001)
            input_path = '/proc/%d/status' % posix.getpid()
            with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
                contents = f.read()
                f2.write(contents)
                log('Wrote %s to %s (--parser-mem-dump)', input_path,
                    opts.parser_mem_dump)

        ui.PrintAst(nodes_out, opts)

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if opts.runtime_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % posix.getpid()
        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
            contents = f.read()
            f2.write(contents)
            log('Wrote %s to %s (--runtime-mem-dump)', input_path,
                opts.runtime_mem_dump)

    # NOTE: We haven't closed the file opened with fd_state.Open
    return status
Beispiel #21
0
from core import util
from core import test_lib

Process = process.Process
ExternalThunk = process.ExternalThunk

log = util.log


def Banner(msg):
    print('-' * 60)
    print(msg)


_WAITER = process.Waiter()
_FD_STATE = process.FdState()
_EXT_PROG = process.ExternalProgram(False, _FD_STATE, util.NullDebugFile())


def _CommandNode(code_str, arena):
    c_parser = test_lib.InitCommandParser(code_str, arena=arena)
    return c_parser.ParseLogicalLine()


def _ExtProc(argv):
    return Process(ExternalThunk(_EXT_PROG, argv, {}))


class ProcessTest(unittest.TestCase):
    def testStdinRedirect(self):
        waiter = process.Waiter()
Beispiel #22
0
def InitExecutor(parse_ctx=None,
                 comp_lookup=None,
                 arena=None,
                 mem=None,
                 aliases=None,
                 ext_prog=None):
    if parse_ctx:
        arena = parse_ctx.arena
        parse_opts = parse_ctx.parse_opts
    else:
        arena or MakeArena('<InitExecutor>')
        parse_opts = parse_lib.OilParseOptions()
        parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, None)

    mem = mem or state.Mem('', [], {}, arena)
    exec_opts = state.ExecOpts(mem, parse_opts, None)
    errfmt = ui.ErrorFormatter(arena)
    job_state = process.JobState()
    fd_state = process.FdState(errfmt, job_state)
    funcs = {}
    aliases = {} if aliases is None else aliases

    compopt_state = completion.OptionState()
    comp_lookup = comp_lookup or completion.Lookup()

    readline = None  # simulate not having it
    new_var = builtin_assign.NewVar(mem, funcs, errfmt)
    builtins = {  # Lookup
        builtin_e.ECHO: builtin_pure.Echo(exec_opts),
        builtin_e.SHIFT: builtin_assign.Shift(mem),

        builtin_e.HISTORY: builtin.History(readline),

        builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state, errfmt),
        builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),

        builtin_e.ALIAS: builtin_pure.Alias(aliases, errfmt),
        builtin_e.UNALIAS: builtin_pure.UnAlias(aliases, errfmt),

        builtin_e.DECLARE: new_var,
        builtin_e.TYPESET: new_var,
        builtin_e.LOCAL: new_var,

        builtin_e.EXPORT: builtin_assign.Export(mem, errfmt),
        builtin_e.READONLY: builtin_assign.Readonly(mem, errfmt),
    }

    # For the tests, we do not use 'readline'.
    exec_opts = state.ExecOpts(mem, parse_opts, None)

    debug_f = util.DebugFile(sys.stderr)
    exec_deps = cmd_exec.Deps()
    exec_deps.search_path = state.SearchPath(mem)
    exec_deps.errfmt = errfmt
    exec_deps.trap_nodes = []
    exec_deps.job_state = job_state
    exec_deps.waiter = process.Waiter(exec_deps.job_state, exec_opts)

    exec_deps.ext_prog = \
        ext_prog or process.ExternalProgram('', fd_state,
                                            exec_deps.search_path, errfmt,
                                            debug_f)

    exec_deps.dumper = dev.CrashDumper('')
    exec_deps.debug_f = debug_f
    exec_deps.trace_f = debug_f

    splitter = split.SplitContext(mem)
    exec_deps.splitter = splitter

    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
    exec_deps.word_ev = word_ev

    arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.arith_ev = arith_ev

    word_ev.arith_ev = arith_ev  # Circular

    bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
    exec_deps.bool_ev = bool_ev

    tracer = dev.Tracer(parse_ctx, exec_opts, mem, word_ev, debug_f)
    exec_deps.tracer = tracer

    ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
                           parse_ctx, exec_deps)

    spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
                                            comp_lookup)
    # Add some builtins that depend on the executor!
    complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
    builtins[builtin_e.COMPLETE] = complete_builtin
    builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)

    return ex
Beispiel #23
0

# TODO: Put these all in a function.
_ARENA = test_lib.MakeArena('process_test.py')

_MEM = state.Mem('', [], _ARENA, [])
state.InitMem(_MEM, {}, '0.1')

_OPT_ARRAY = [False] * option_i.ARRAY_SIZE
_PARSE_OPTS = optview.Parse(_OPT_ARRAY)
_ERREXIT = state._ErrExit()
_EXEC_OPTS = state.MutableOpts(_MEM, _OPT_ARRAY, _ERREXIT, None)
_JOB_STATE = process.JobState()
_WAITER = process.Waiter(_JOB_STATE, _EXEC_OPTS)
_ERRFMT = ui.ErrorFormatter(_ARENA)
_FD_STATE = process.FdState(_ERRFMT, _JOB_STATE)
_EXT_PROG = process.ExternalProgram('', _FD_STATE, _ERRFMT,
                                    util.NullDebugFile())


def _CommandNode(code_str, arena):
  c_parser = test_lib.InitCommandParser(code_str, arena=arena)
  return c_parser.ParseLogicalLine()


def _ExtProc(argv):
  arg_vec = cmd_value.Argv(argv, [0] * len(argv))
  argv0_path = None
  for path_entry in ['/bin', '/usr/bin']:
    full_path = os.path.join(path_entry, argv[0])
    if os.path.exists(full_path):
Beispiel #24
0
def Main(lang, arg_r, environ, login_shell, loader, line_input):
    # type: (str, args.Reader, Dict[str, str], bool, pyutil._ResourceLoader, Any) -> int
    """The full shell lifecycle.  Used by bin/osh and bin/oil.

  Args:
    lang: 'osh' or 'oil'
    argv0, arg_r: command line arguments
    environ: environment
    login_shell: Was - on the front?
    loader: to get help, version, grammar, etc.
    line_input: optional GNU readline
  """
    # Differences between osh and oil:
    # - --help?  I guess Oil has a SUPERSET of OSH options.
    # - oshrc vs oilrc
    # - shopt -s oil:all
    # - Change the prompt in the interactive shell?

    # osh-pure:
    # - no oil grammar
    # - no expression evaluator
    # - no interactive shell, or line_input
    # - no process.*
    #   process.{ExternalProgram,Waiter,FdState,JobState,SignalState} -- we want
    #   to evaluate config files without any of these
    # Modules not translated yet: completion, comp_ui, builtin_comp, process
    # - word evaluator
    #   - shouldn't glob?  set -o noglob?  or hard failure?
    #   - ~ shouldn't read from the file system
    #     - I guess it can just be the HOME=HOME?
    # Builtin:
    #   shellvm -c 'echo hi'
    #   shellvm <<< 'echo hi'

    argv0 = arg_r.Peek()
    assert argv0 is not None
    arg_r.Next()

    assert lang in ('osh', 'oil'), lang

    try:
        attrs = flag_spec.ParseMore('main', arg_r)
    except error.Usage as e:
        stderr_line('osh usage error: %s', e.msg)
        return 2
    flag = arg_types.main(attrs.attrs)

    arena = alloc.Arena()
    errfmt = ui.ErrorFormatter(arena)

    help_builtin = builtin_misc.Help(loader, errfmt)
    if flag.help:
        help_builtin.Run(pure.MakeBuiltinArgv(['%s-usage' % lang]))
        return 0
    if flag.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        pyutil.ShowAppVersion('Oil', loader)
        return 0

    no_str = None  # type: str

    debug_stack = []  # type: List[state.DebugFrame]
    if arg_r.AtEnd():
        dollar0 = argv0
    else:
        dollar0 = arg_r.Peek()  # the script name, or the arg after -c

        # Copy quirky bash behavior.
        frame0 = state.DebugFrame(dollar0, 'main', no_str, state.LINE_ZERO, 0,
                                  0)
        debug_stack.append(frame0)

    # Copy quirky bash behavior.
    frame1 = state.DebugFrame(no_str, no_str, no_str, runtime.NO_SPID, 0, 0)
    debug_stack.append(frame1)

    script_name = arg_r.Peek()  # type: Optional[str]
    arg_r.Next()
    mem = state.Mem(dollar0, arg_r.Rest(), arena, debug_stack)

    version_str = pyutil.GetVersion(loader)
    state.InitMem(mem, environ, version_str)

    builtin_funcs.Init(mem)

    procs = {}  # type: Dict[str, command__ShFunction]

    job_state = process.JobState()
    fd_state = process.FdState(errfmt, job_state, mem)

    opt_hook = ShellOptHook(line_input)
    parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
    # TODO: only MutableOpts needs mem, so it's not a true circular dep.
    mem.exec_opts = exec_opts  # circular dep

    if attrs.show_options:  # special case: sh -o
        mutable_opts.ShowOptions([])
        return 0

    # Set these BEFORE processing flags, so they can be overridden.
    if lang == 'oil':
        mutable_opts.SetShoptOption('oil:all', True)

    builtin_pure.SetShellOpts(mutable_opts, attrs.opt_changes,
                              attrs.shopt_changes)
    # feedback between runtime and parser
    aliases = {}  # type: Dict[str, str]

    oil_grammar = pyutil.LoadOilGrammar(loader)

    if flag.one_pass_parse and not exec_opts.noexec():
        raise error.Usage('--one-pass-parse requires noexec (-n)')
    parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
    parse_ctx.Init_OnePassParse(flag.one_pass_parse)

    # Three ParseContext instances SHARE aliases.
    comp_arena = alloc.Arena()
    comp_arena.PushSource(source.Unused('completion'))
    trail1 = parse_lib.Trail()
    # one_pass_parse needs to be turned on to complete inside backticks.  TODO:
    # fix the issue where ` gets erased because it's not part of
    # set_completer_delims().
    comp_ctx = parse_lib.ParseContext(comp_arena, parse_opts, aliases,
                                      oil_grammar)
    comp_ctx.Init_Trail(trail1)
    comp_ctx.Init_OnePassParse(True)

    hist_arena = alloc.Arena()
    hist_arena.PushSource(source.Unused('history'))
    trail2 = parse_lib.Trail()
    hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
                                      oil_grammar)
    hist_ctx.Init_Trail(trail2)

    # Deps helps manages dependencies.  These dependencies are circular:
    # - cmd_ev and word_ev, arith_ev -- for command sub, arith sub
    # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
    # - cmd_ev and builtins (which execute code, like eval)
    # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
    cmd_deps = cmd_eval.Deps()
    cmd_deps.mutable_opts = mutable_opts

    # TODO: In general, cmd_deps are shared between the mutually recursive
    # evaluators.  Some of the four below are only shared between a builtin and
    # the CommandEvaluator, so we could put them somewhere else.
    cmd_deps.traps = {}
    cmd_deps.trap_nodes = []  # TODO: Clear on fork() to avoid duplicates

    waiter = process.Waiter(job_state, exec_opts)

    my_pid = posix.getpid()

    debug_path = ''
    debug_dir = environ.get('OSH_DEBUG_DIR')
    if flag.debug_file is not None:
        # --debug-file takes precedence over OSH_DEBUG_DIR
        debug_path = flag.debug_file
    elif debug_dir is not None:
        debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)

    if len(debug_path):
        # This will be created as an empty file if it doesn't exist, or it could be
        # a pipe.
        try:
            debug_f = util.DebugFile(
                fd_state.OpenForWrite(debug_path))  # type: util._DebugFile
        except OSError as e:
            stderr_line("osh: Couldn't open %r: %s", debug_path,
                        posix.strerror(e.errno))
            return 2
    else:
        debug_f = util.NullDebugFile()

    cmd_deps.debug_f = debug_f

    # Not using datetime for dependency reasons.  TODO: maybe show the date at
    # the beginning of the log, and then only show time afterward?  To save
    # space, and make space for microseconds.  (datetime supports microseconds
    # but time.strftime doesn't).
    if mylib.PYTHON:
        iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
        debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid,
                    arg_r.argv)
    if len(debug_path):
        debug_f.log('Writing logs to %r', debug_path)

    interp = environ.get('OSH_HIJACK_SHEBANG', '')
    search_path = state.SearchPath(mem)
    ext_prog = process.ExternalProgram(interp, fd_state, errfmt, debug_f)

    splitter = split.SplitContext(mem)

    # split() builtin
    # TODO: Accept IFS as a named arg?  split('a b', IFS=' ')
    builtin_funcs.SetGlobalFunc(
        mem,
        'split',
        lambda s, ifs=None: splitter.SplitForWordEval(s, ifs=ifs))

    # glob() builtin
    # TODO: This is instantiation is duplicated in osh/word_eval.py
    globber = glob_.Globber(exec_opts)
    builtin_funcs.SetGlobalFunc(mem, 'glob', lambda s: globber.OilFuncCall(s))

    # This could just be OSH_DEBUG_STREAMS='debug crash' ?  That might be
    # stuffing too much into one, since a .json crash dump isn't a stream.
    crash_dump_dir = environ.get('OSH_CRASH_DUMP_DIR', '')
    cmd_deps.dumper = dev.CrashDumper(crash_dump_dir)

    if flag.xtrace_to_debug_file:
        trace_f = debug_f
    else:
        trace_f = util.DebugFile(mylib.Stderr())

    comp_lookup = completion.Lookup()

    # Various Global State objects to work around readline interfaces
    compopt_state = completion.OptionState()
    comp_ui_state = comp_ui.State()
    prompt_state = comp_ui.PromptState()

    dir_stack = state.DirStack()

    #
    # Initialize builtins that don't depend on evaluators
    #

    builtins = {}  # type: Dict[int, vm._Builtin]

    pure.AddPure(builtins, mem, procs, mutable_opts, aliases, search_path,
                 errfmt)
    pure.AddIO(builtins, mem, dir_stack, exec_opts, splitter, errfmt)
    AddProcess(builtins, mem, ext_prog, fd_state, job_state, waiter,
               search_path, errfmt)
    AddOil(builtins, mem, errfmt)

    builtins[builtin_i.help] = help_builtin

    # Interactive, depend on line_input
    builtins[builtin_i.bind] = builtin_lib.Bind(line_input, errfmt)
    builtins[builtin_i.history] = builtin_lib.History(line_input,
                                                      mylib.Stdout())

    #
    # Assignment builtins
    #

    assign_b = {}  # type: Dict[int, vm._AssignBuiltin]

    new_var = builtin_assign.NewVar(mem, procs, errfmt)
    assign_b[builtin_i.declare] = new_var
    assign_b[builtin_i.typeset] = new_var
    assign_b[builtin_i.local] = new_var

    assign_b[builtin_i.export_] = builtin_assign.Export(mem, errfmt)
    assign_b[builtin_i.readonly] = builtin_assign.Readonly(mem, errfmt)

    #
    # Initialize Evaluators
    #

    arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, parse_ctx, errfmt)
    bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, parse_ctx, errfmt)
    expr_ev = expr_eval.OilEvaluator(mem, procs, splitter, errfmt)
    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)
    cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
                                       arena, cmd_deps)

    shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
                                      builtins, search_path, ext_prog, waiter,
                                      job_state, fd_state, errfmt)

    # PromptEvaluator rendering is needed in non-interactive shells for @P.
    prompt_ev = prompt.Evaluator(lang, parse_ctx, mem)
    tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
                        trace_f)

    # Wire up circular dependencies.
    vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
                        prompt_ev, tracer)

    #
    # Initialize builtins that depend on evaluators
    #

    # note: 'printf -v a[i]' and 'unset a[i]' require same deps
    builtins[builtin_i.printf] = builtin_printf.Printf(mem, exec_opts,
                                                       parse_ctx, arith_ev,
                                                       errfmt)
    builtins[builtin_i.unset] = builtin_assign.Unset(mem, exec_opts, procs,
                                                     parse_ctx, arith_ev,
                                                     errfmt)
    builtins[builtin_i.eval] = builtin_meta.Eval(parse_ctx, exec_opts, cmd_ev)

    source_builtin = builtin_meta.Source(parse_ctx, search_path, cmd_ev,
                                         fd_state, errfmt)
    builtins[builtin_i.source] = source_builtin
    builtins[builtin_i.dot] = source_builtin

    builtins[builtin_i.builtin] = builtin_meta.Builtin(shell_ex, errfmt)
    builtins[builtin_i.command] = builtin_meta.Command(shell_ex, procs,
                                                       aliases, search_path)

    spec_builder = builtin_comp.SpecBuilder(cmd_ev, parse_ctx, word_ev,
                                            splitter, comp_lookup)
    complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
    builtins[builtin_i.complete] = complete_builtin
    builtins[builtin_i.compgen] = builtin_comp.CompGen(spec_builder)
    builtins[builtin_i.compopt] = builtin_comp.CompOpt(compopt_state, errfmt)
    builtins[builtin_i.compadjust] = builtin_comp.CompAdjust(mem)

    # These builtins take blocks, and thus need cmd_ev.
    builtins[builtin_i.cd] = builtin_misc.Cd(mem, dir_stack, cmd_ev, errfmt)
    builtins[builtin_i.json] = builtin_oil.Json(mem, cmd_ev, errfmt)

    sig_state = pyos.SignalState()
    sig_state.InitShell()

    builtins[builtin_i.trap] = builtin_process.Trap(sig_state, cmd_deps.traps,
                                                    cmd_deps.trap_nodes,
                                                    parse_ctx, errfmt)

    # History evaluation is a no-op if line_input is None.
    hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)

    if flag.c is not None:
        arena.PushSource(source.CFlag())
        line_reader = reader.StringLineReader(flag.c,
                                              arena)  # type: reader._Reader
        if flag.i:  # -c and -i can be combined
            mutable_opts.set_interactive()

    elif flag.i:  # force interactive
        arena.PushSource(source.Stdin(' -i'))
        line_reader = py_reader.InteractiveLineReader(arena, prompt_ev,
                                                      hist_ev, line_input,
                                                      prompt_state)
        mutable_opts.set_interactive()

    else:
        if script_name is None:
            stdin = mylib.Stdin()
            if stdin.isatty():
                arena.PushSource(source.Interactive())
                line_reader = py_reader.InteractiveLineReader(
                    arena, prompt_ev, hist_ev, line_input, prompt_state)
                mutable_opts.set_interactive()
            else:
                arena.PushSource(source.Stdin(''))
                line_reader = reader.FileLineReader(stdin, arena)
        else:
            arena.PushSource(source.MainFile(script_name))
            try:
                f = fd_state.Open(script_name)
            except OSError as e:
                stderr_line("osh: Couldn't open %r: %s", script_name,
                            posix.strerror(e.errno))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    c_parser = parse_ctx.MakeOshParser(line_reader)

    if exec_opts.interactive():
        # bash: 'set -o emacs' is the default only in the interactive shell
        mutable_opts.set_emacs()

        # Calculate ~/.config/oil/oshrc or oilrc
        # Use ~/.config/oil to avoid cluttering the user's home directory.  Some
        # users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.

        # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
        home_dir = pyos.GetMyHomeDir()
        assert home_dir is not None
        history_filename = os_path.join(home_dir,
                                        '.config/oil/history_%s' % lang)

        if line_input:
            # NOTE: We're using a different WordEvaluator here.
            ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
                                                   errfmt)

            ev.arith_ev = arith_ev
            ev.expr_ev = expr_ev
            ev.prompt_ev = prompt_ev
            ev.CheckCircularDeps()

            root_comp = completion.RootCompleter(ev, mem, comp_lookup,
                                                 compopt_state, comp_ui_state,
                                                 comp_ctx, debug_f)

            term_width = 0
            if flag.completion_display == 'nice':
                try:
                    term_width = libc.get_terminal_width()
                except IOError:  # stdin not a terminal
                    pass

            if term_width != 0:
                display = comp_ui.NiceDisplay(
                    term_width, comp_ui_state, prompt_state, debug_f,
                    line_input)  # type: comp_ui._IDisplay
            else:
                display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                                 debug_f)

            comp_ui.InitReadline(line_input, history_filename, root_comp,
                                 display, debug_f)
            _InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup)

        else:  # Without readline module
            display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                             debug_f)

        sig_state.InitInteractiveShell(display)

        rc_path = flag.rcfile or os_path.join(home_dir,
                                              '.config/oil/%src' % lang)
        try:
            # NOTE: Should be called AFTER _InitDefaultCompletions.
            SourceStartupFile(fd_state, rc_path, lang, parse_ctx, cmd_ev)
        except util.UserExit as e:
            return e.status

        line_reader.Reset()  # After sourcing startup file, render $PS1

        prompt_plugin = prompt.UserPlugin(mem, parse_ctx, cmd_ev)
        try:
            status = main_loop.Interactive(flag, cmd_ev, c_parser, display,
                                           prompt_plugin, errfmt)
        except util.UserExit as e:
            status = e.status
        box = [status]
        cmd_ev.MaybeRunExitTrap(box)
        status = box[0]

        return status

    if flag.rcfile:  # bash doesn't have this warning, but it's useful
        stderr_line('osh warning: --rcfile ignored in non-interactive shell')

    if exec_opts.noexec():
        status = 0
        try:
            node = main_loop.ParseWholeFile(c_parser)
        except error.Parse as e:
            ui.PrettyPrintError(e, arena)
            status = 2

        if status == 0:
            if flag.parser_mem_dump is not None:  # only valid in -n mode
                input_path = '/proc/%d/status' % posix.getpid()
                pyutil.CopyFile(input_path, flag.parser_mem_dump)

            ui.PrintAst(node, flag)
    else:
        if flag.parser_mem_dump is not None:
            raise error.Usage('--parser-mem-dump can only be used with -n')

        try:
            status = main_loop.Batch(cmd_ev,
                                     c_parser,
                                     arena,
                                     cmd_flags=cmd_eval.IsMainProgram)
        except util.UserExit as e:
            status = e.status
        box = [status]
        cmd_ev.MaybeRunExitTrap(box)
        status = box[0]

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if flag.runtime_mem_dump is not None:
        input_path = '/proc/%d/status' % posix.getpid()
        pyutil.CopyFile(input_path, flag.runtime_mem_dump)

    # NOTE: We haven't closed the file opened with fd_state.Open
    return status
Beispiel #25
0
def OshMain(argv0, argv, login_shell):

    arg_r = args.Reader(argv)
    try:
        opts = OSH_SPEC.Parse(arg_r)
    except args.UsageError as e:
        ui.usage('osh usage error: %s', e)
        return 2

    if opts.help:
        loader = util.GetResourceLoader()
        builtin.Help(['osh-usage'], loader)
        return 0
    if opts.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        _ShowVersion()
        return 0

    # TODO: This should be in interactive mode only?
    builtin.RegisterSigIntHandler()

    if arg_r.AtEnd():
        dollar0 = argv0
        has_main = False
    else:
        dollar0 = arg_r.Peek()  # the script name, or the arg after -c
        has_main = True

    pool = alloc.Pool()
    arena = pool.NewArena()

    # NOTE: has_main is only for ${BASH_SOURCE[@} and family.  Could be a
    # required arg.
    mem = state.Mem(dollar0,
                    argv[arg_r.i + 1:],
                    posix.environ,
                    arena,
                    has_main=has_main)
    funcs = {}

    comp_lookup = completion.CompletionLookup()

    fd_state = process.FdState()
    exec_opts = state.ExecOpts(mem, readline)
    builtin.SetExecOpts(exec_opts, opts.opt_changes)
    aliases = {}  # feedback between runtime and parser

    parse_ctx = parse_lib.ParseContext(arena, aliases)

    if opts.debug_file:
        debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w'))
    else:
        debug_f = util.NullDebugFile()
    debug_f.log('Debug file is %s', opts.debug_file)

    # Controlled by env variable, flag, or hook?
    dumper = dev.CrashDumper(posix.environ.get('OSH_CRASH_DUMP_DIR', ''))
    if opts.xtrace_to_debug_file:
        trace_f = debug_f
    else:
        trace_f = util.DebugFile(sys.stderr)
    devtools = dev.DevTools(dumper, debug_f, trace_f)

    ex = cmd_exec.Executor(mem, fd_state, funcs, comp_lookup, exec_opts,
                           parse_ctx, devtools)

    # NOTE: The rc file can contain both commands and functions... ideally we
    # would only want to save nodes/lines for the functions.
    try:
        rc_path = 'oilrc'
        arena.PushSource(rc_path)
        with open(rc_path) as f:
            rc_line_reader = reader.FileLineReader(f, arena)
            _, rc_c_parser = parse_ctx.MakeParser(rc_line_reader)
            try:
                status = main_loop.Batch(ex, rc_c_parser, arena)
            finally:
                arena.PopSource()
    except IOError as e:
        if e.errno != errno.ENOENT:
            raise

    # Needed in non-interactive shells for @P
    prompt = ui.Prompt(arena, parse_ctx, ex)
    ui.PROMPT = prompt

    if opts.c is not None:
        arena.PushSource('<command string>')
        line_reader = reader.StringLineReader(opts.c, arena)
        if opts.i:  # -c and -i can be combined
            exec_opts.interactive = True
    elif opts.i:  # force interactive
        arena.PushSource('<stdin -i>')
        line_reader = reader.InteractiveLineReader(arena, prompt)
        exec_opts.interactive = True
    else:
        try:
            script_name = arg_r.Peek()
        except IndexError:
            if sys.stdin.isatty():
                arena.PushSource('<interactive>')
                line_reader = reader.InteractiveLineReader(arena, prompt)
                exec_opts.interactive = True
            else:
                arena.PushSource('<stdin>')
                line_reader = reader.FileLineReader(sys.stdin, arena)
        else:
            arena.PushSource(script_name)
            try:
                f = fd_state.Open(script_name)
            except OSError as e:
                util.error("Couldn't open %r: %s", script_name,
                           posix.strerror(e.errno))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    w_parser, c_parser = parse_ctx.MakeParser(line_reader)

    if exec_opts.interactive:
        # NOTE: We're using a different evaluator here.  The completion system can
        # also run functions... it gets the Executor through Executor._Complete.
        if HAVE_READLINE:
            splitter = legacy.SplitContext(mem)  # TODO: share with executor.
            ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
                                                   arena)
            progress_f = ui.StatusLine()
            var_action = completion.VariablesActionInternal(ex.mem)
            root_comp = completion.RootCompleter(ev, comp_lookup, var_action,
                                                 parse_ctx, progress_f,
                                                 debug_f)
            completion.Init(readline, root_comp, debug_f)
            _InitDefaultCompletions(ex, comp_lookup)

        return main_loop.Interactive(opts, ex, c_parser, arena)

    # TODO: Remove this after removing it from benchmarks/osh-runtime.  It's no
    # longer relevant with main_loop.
    if opts.parser_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % posix.getpid()
        with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
            contents = f.read()
            f2.write(contents)
            log('Wrote %s to %s (--parser-mem-dump)', input_path,
                opts.parser_mem_dump)

    nodes_out = [] if exec_opts.noexec else None

    _tlog('Execute(node)')
    status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)

    if nodes_out is not None:
        ui.PrintAst(nodes_out, opts)

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if opts.runtime_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % posix.getpid()
        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
            contents = f.read()
            f2.write(contents)
            log('Wrote %s to %s (--runtime-mem-dump)', input_path,
                opts.runtime_mem_dump)

    # NOTE: We haven't closed the file opened with fd_state.Open
    return status
Beispiel #26
0
def OshMain(argv0, argv, login_shell):
    spec = args.FlagsAndOptions()
    spec.ShortFlag('-c', args.Str, quit_parsing_flags=True)  # command string
    spec.ShortFlag('-i')  # interactive

    # TODO: -h too
    spec.LongFlag('--help')
    spec.LongFlag('--version')
    spec.LongFlag(
        '--ast-format',
        ['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap', 'none'],
        default='abbrev-text')
    spec.LongFlag('--show-ast')  # execute and show
    spec.LongFlag('--fix')
    spec.LongFlag('--debug-spans')  # For oshc translate
    spec.LongFlag('--print-status')
    spec.LongFlag(
        '--trace',
        ['cmd-parse', 'word-parse', 'lexer'])  # NOTE: can only trace one now
    spec.LongFlag('--hijack-shebang')

    # For benchmarks/*.sh
    spec.LongFlag('--parser-mem-dump', args.Str)
    spec.LongFlag('--runtime-mem-dump', args.Str)

    builtin.AddOptionsToArgSpec(spec)

    try:
        opts, opt_index = spec.Parse(argv)
    except args.UsageError as e:
        util.usage(str(e))
        return 2

    if opts.help:
        loader = util.GetResourceLoader()
        builtin.Help(['osh-usage'], loader)
        return 0
    if opts.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        _ShowVersion()
        return 0

    trace_state = util.TraceState()
    if 'cmd-parse' == opts.trace:
        util.WrapMethods(cmd_parse.CommandParser, trace_state)
    if 'word-parse' == opts.trace:
        util.WrapMethods(word_parse.WordParser, trace_state)
    if 'lexer' == opts.trace:
        util.WrapMethods(lexer.Lexer, trace_state)

    if opt_index == len(argv):
        dollar0 = argv0
    else:
        dollar0 = argv[opt_index]  # the script name, or the arg after -c

    # TODO: Create a --parse action or 'osh parse' or 'oil osh-parse'
    # osh-fix
    # It uses a different memory-management model.  It's a batch program and not
    # an interactive program.

    pool = alloc.Pool()
    arena = pool.NewArena()

    # TODO: Maybe wrap this initialization sequence up in an oil_State, like
    # lua_State.
    status_lines = ui.MakeStatusLines()
    mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ, arena)
    funcs = {}

    # Passed to Executor for 'complete', and passed to completion.Init
    if completion:
        comp_lookup = completion.CompletionLookup()
    else:
        # TODO: NullLookup?
        comp_lookup = None

    exec_opts = state.ExecOpts(mem)
    builtin.SetExecOpts(exec_opts, opts.opt_changes)

    fd_state = process.FdState()
    ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, readline,
                           completion, comp_lookup, exec_opts, arena)

    # NOTE: The rc file can contain both commands and functions... ideally we
    # would only want to save nodes/lines for the functions.
    try:
        rc_path = 'oilrc'
        arena.PushSource(rc_path)
        with open(rc_path) as f:
            rc_line_reader = reader.FileLineReader(f, arena)
            _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
            try:
                rc_node = rc_c_parser.ParseWholeFile()
                if not rc_node:
                    err = rc_c_parser.Error()
                    ui.PrintErrorStack(err, arena, sys.stderr)
                    return 2  # parse error is code 2
            finally:
                arena.PopSource()

        status = ex.Execute(rc_node)
        #print('oilrc:', status, cflow, file=sys.stderr)
        # Ignore bad status?
    except IOError as e:
        if e.errno != errno.ENOENT:
            raise

    if opts.c is not None:
        arena.PushSource('<command string>')
        line_reader = reader.StringLineReader(opts.c, arena)
        if opts.i:  # -c and -i can be combined
            exec_opts.interactive = True
    elif opts.i:  # force interactive
        arena.PushSource('<stdin -i>')
        line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
        exec_opts.interactive = True
    else:
        try:
            script_name = argv[opt_index]
        except IndexError:
            if sys.stdin.isatty():
                arena.PushSource('<interactive>')
                line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
                exec_opts.interactive = True
            else:
                arena.PushSource('<stdin>')
                line_reader = reader.FileLineReader(sys.stdin, arena)
        else:
            arena.PushSource(script_name)
            try:
                f = fd_state.Open(script_name)
            except OSError as e:
                util.error("Couldn't open %r: %s", script_name,
                           os.strerror(e.errno))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    w_parser, c_parser = parse_lib.MakeParser(line_reader, arena)

    if exec_opts.interactive:
        # NOTE: We're using a different evaluator here.  The completion system can
        # also run functions... it gets the Executor through Executor._Complete.
        if HAVE_READLINE:
            splitter = legacy.SplitContext(mem)
            ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
            status_out = completion.StatusOutput(status_lines, exec_opts)
            completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
                            status_out, ev)

        return InteractiveLoop(opts, ex, c_parser, w_parser, line_reader)
    else:
        # Parse the whole thing up front
        #print('Parsing file')

        _tlog('ParseWholeFile')
        # TODO: Do I need ParseAndEvalLoop?  How is it different than
        # InteractiveLoop?
        try:
            node = c_parser.ParseWholeFile()
        except util.ParseError as e:
            ui.PrettyPrintError(e, arena, sys.stderr)
            print('parse error: %s' % e.UserErrorString(), file=sys.stderr)
            return 2
        else:
            # TODO: Remove this older form of error handling.
            if not node:
                err = c_parser.Error()
                assert err, err  # can't be empty
                ui.PrintErrorStack(err, arena, sys.stderr)
                return 2  # parse error is code 2

        do_exec = True
        if opts.fix:
            #log('SPANS: %s', arena.spans)
            osh2oil.PrintAsOil(arena, node, opts.debug_spans)
            do_exec = False
        if exec_opts.noexec:
            do_exec = False

        # Do this after parsing the entire file.  There could be another option to
        # do it before exiting runtime?
        if opts.parser_mem_dump:
            # This might be superstition, but we want to let the value stabilize
            # after parsing.  bash -c 'cat /proc/$$/status' gives different results
            # with a sleep.
            time.sleep(0.001)
            input_path = '/proc/%d/status' % os.getpid()
            with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
                contents = f.read()
                f2.write(contents)
                log('Wrote %s to %s (--parser-mem-dump)', input_path,
                    opts.parser_mem_dump)

        # -n prints AST, --show-ast prints and executes
        if exec_opts.noexec or opts.show_ast:
            if opts.ast_format == 'none':
                print('AST not printed.', file=sys.stderr)
            elif opts.ast_format == 'oheap':
                # TODO: Make this a separate flag?
                if sys.stdout.isatty():
                    raise RuntimeError(
                        'ERROR: Not dumping binary data to a TTY.')
                f = sys.stdout

                enc = encode.Params()
                out = encode.BinOutput(f)
                encode.EncodeRoot(node, enc, out)

            else:  # text output
                f = sys.stdout

                if opts.ast_format in ('text', 'abbrev-text'):
                    ast_f = fmt.DetectConsoleOutput(f)
                elif opts.ast_format in ('html', 'abbrev-html'):
                    ast_f = fmt.HtmlOutput(f)
                else:
                    raise AssertionError
                abbrev_hook = (ast_lib.AbbreviateNodes
                               if 'abbrev-' in opts.ast_format else None)
                tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
                ast_f.FileHeader()
                fmt.PrintTree(tree, ast_f)
                ast_f.FileFooter()
                ast_f.write('\n')

            #util.log("Execution skipped because 'noexec' is on ")
            status = 0

        if do_exec:
            _tlog('Execute(node)')
            status = ex.ExecuteAndRunExitTrap(node)
            # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
            # don't because they call sys.exit().
            if opts.runtime_mem_dump:
                # This might be superstition, but we want to let the value stabilize
                # after parsing.  bash -c 'cat /proc/$$/status' gives different results
                # with a sleep.
                time.sleep(0.001)
                input_path = '/proc/%d/status' % os.getpid()
                with open(input_path) as f, open(opts.runtime_mem_dump,
                                                 'w') as f2:
                    contents = f.read()
                    f2.write(contents)
                    log('Wrote %s to %s (--runtime-mem-dump)', input_path,
                        opts.runtime_mem_dump)

        else:
            status = 0

    return status
Beispiel #27
0
def ShellMain(lang, argv0, argv, login_shell):
    """Used by bin/osh and bin/oil.

  Args:
    lang: 'osh' or 'oil'
    argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'.  Like busybox.
    login_shell: Was - on the front?
  """
    # Differences between osh and oil:
    # - --help?  I guess Oil has a SUPERSET of OSH options.
    # - oshrc vs oilrc
    # - the parser and executor
    # - Change the prompt in the interactive shell?

    assert lang in ('osh', 'oil'), lang

    arg_r = args.Reader(argv)
    try:
        opts = OSH_SPEC.Parse(arg_r)
    except args.UsageError as e:
        ui.Stderr('osh usage error: %s', e.msg)
        return 2

    # NOTE: This has a side effect of deleting _OVM_* from the environment!
    # TODO: Thread this throughout the program, and get rid of the global
    # variable in core/util.py.  Rename to InitResourceLaoder().  It's now only
    # used for the 'help' builtin and --version.
    loader = pyutil.GetResourceLoader()

    if opts.help:
        builtin.Help(['%s-usage' % lang], loader)
        return 0
    if opts.version:
        # OSH version is the only binary in Oil right now, so it's all one version.
        _ShowVersion()
        return 0

    if arg_r.AtEnd():
        dollar0 = argv0
        has_main = False
    else:
        dollar0 = arg_r.Peek()  # the script name, or the arg after -c
        has_main = True

    arena = alloc.Arena()
    errfmt = ui.ErrorFormatter(arena)

    # NOTE: has_main is only for ${BASH_SOURCE[@} and family.  Could be a
    # required arg.
    mem = state.Mem(dollar0,
                    argv[arg_r.i + 1:],
                    posix.environ,
                    arena,
                    has_main=has_main)
    builtin_funcs.Init(mem)

    procs = {}

    job_state = process.JobState()
    fd_state = process.FdState(errfmt, job_state)
    parse_opts = parse_lib.OilParseOptions()
    exec_opts = state.ExecOpts(mem, parse_opts, line_input)

    if opts.show_options:  # special case: sh -o
        exec_opts.ShowOptions([])
        return 0

    builtin_pure.SetExecOpts(exec_opts, opts.opt_changes, opts.shopt_changes)
    aliases = {}  # feedback between runtime and parser

    oil_grammar = meta.LoadOilGrammar(loader)

    if opts.one_pass_parse and not exec_opts.noexec:
        raise args.UsageError('--one-pass-parse requires noexec (-n)')
    parse_ctx = parse_lib.ParseContext(arena,
                                       parse_opts,
                                       aliases,
                                       oil_grammar,
                                       one_pass_parse=opts.one_pass_parse)

    # Three ParseContext instances SHARE aliases.
    comp_arena = alloc.Arena()
    comp_arena.PushSource(source.Unused('completion'))
    trail1 = parse_lib.Trail()
    # one_pass_parse needs to be turned on to complete inside backticks.  TODO:
    # fix the issue where ` gets erased because it's not part of
    # set_completer_delims().
    comp_ctx = parse_lib.ParseContext(comp_arena,
                                      parse_opts,
                                      aliases,
                                      oil_grammar,
                                      trail=trail1,
                                      one_pass_parse=True)

    hist_arena = alloc.Arena()
    hist_arena.PushSource(source.Unused('history'))
    trail2 = parse_lib.Trail()
    hist_ctx = parse_lib.ParseContext(hist_arena,
                                      parse_opts,
                                      aliases,
                                      oil_grammar,
                                      trail=trail2)

    # Deps helps manages dependencies.  These dependencies are circular:
    # - ex and word_ev, arith_ev -- for command sub, arith sub
    # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
    # - ex and builtins (which execute code, like eval)
    # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
    exec_deps = cmd_exec.Deps()

    # TODO: In general, exec_deps are shared between the mutually recursive
    # evaluators.  Some of the four below are only shared between a builtin and
    # the Executor, so we could put them somewhere else.
    exec_deps.traps = {}
    exec_deps.trap_nodes = []  # TODO: Clear on fork() to avoid duplicates

    exec_deps.job_state = job_state
    # note: exec_opts.interactive set later
    exec_deps.waiter = process.Waiter(job_state, exec_opts)
    exec_deps.errfmt = errfmt

    my_pid = posix.getpid()

    debug_path = ''
    debug_dir = posix.environ.get('OSH_DEBUG_DIR')
    if opts.debug_file:  # --debug-file takes precedence over OSH_DEBUG_DIR
        debug_path = opts.debug_file
    elif debug_dir:
        debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)

    if debug_path:
        # This will be created as an empty file if it doesn't exist, or it could be
        # a pipe.
        try:
            debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
        except OSError as e:
            ui.Stderr("osh: Couldn't open %r: %s", debug_path,
                      posix.strerror(e.errno))
            return 2
    else:
        debug_f = util.NullDebugFile()

    exec_deps.debug_f = debug_f

    # Not using datetime for dependency reasons.  TODO: maybe show the date at
    # the beginning of the log, and then only show time afterward?  To save
    # space, and make space for microseconds.  (datetime supports microseconds
    # but time.strftime doesn't).
    iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
    debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
    if debug_path:
        debug_f.log('Writing logs to %r', debug_path)

    interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
    exec_deps.search_path = state.SearchPath(mem)
    exec_deps.ext_prog = process.ExternalProgram(interp, fd_state,
                                                 exec_deps.search_path, errfmt,
                                                 debug_f)

    splitter = split.SplitContext(mem)
    exec_deps.splitter = splitter

    # split() builtin
    builtin_funcs.SetGlobalFunc(mem, 'split',
                                lambda s: splitter.SplitForWordEval(s))

    # This could just be OSH_DEBUG_STREAMS='debug crash' ?  That might be
    # stuffing too much into one, since a .json crash dump isn't a stream.
    crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
    exec_deps.dumper = dev.CrashDumper(crash_dump_dir)

    if opts.xtrace_to_debug_file:
        trace_f = debug_f
    else:
        trace_f = util.DebugFile(sys.stderr)
    exec_deps.trace_f = trace_f

    comp_lookup = completion.Lookup()

    # Various Global State objects to work around readline interfaces
    compopt_state = completion.OptionState()
    comp_ui_state = comp_ui.State()
    prompt_state = comp_ui.PromptState()

    dir_stack = state.DirStack()

    new_var = builtin_assign.NewVar(mem, procs, errfmt)

    builtins = {  # Lookup
        builtin_e.ECHO: builtin_pure.Echo(exec_opts),
        builtin_e.PRINTF: builtin_printf.Printf(mem, parse_ctx, errfmt),

        builtin_e.PUSHD: builtin.Pushd(mem, dir_stack, errfmt),
        builtin_e.POPD: builtin.Popd(mem, dir_stack, errfmt),
        builtin_e.DIRS: builtin.Dirs(mem, dir_stack, errfmt),
        builtin_e.PWD: builtin.Pwd(mem, errfmt),

        builtin_e.READ: builtin.Read(splitter, mem),
        builtin_e.HELP: builtin.Help(loader, errfmt),
        builtin_e.HISTORY: builtin.History(line_input),

        # Completion (more added below)
        builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state, errfmt),
        builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),

        # test / [ differ by need_right_bracket
        builtin_e.TEST: builtin_bracket.Test(False, errfmt),
        builtin_e.BRACKET: builtin_bracket.Test(True, errfmt),

        # Assignment (which are pure)
        builtin_e.DECLARE: new_var,
        builtin_e.TYPESET: new_var,
        builtin_e.LOCAL: new_var,

        builtin_e.EXPORT: builtin_assign.Export(mem, errfmt),
        builtin_e.READONLY: builtin_assign.Readonly(mem, errfmt),

        builtin_e.UNSET: builtin_assign.Unset(mem, procs, errfmt),
        builtin_e.SHIFT: builtin_assign.Shift(mem),

        # Pure
        builtin_e.SET: builtin_pure.Set(exec_opts, mem),
        builtin_e.SHOPT: builtin_pure.Shopt(exec_opts),

        builtin_e.ALIAS: builtin_pure.Alias(aliases, errfmt),
        builtin_e.UNALIAS: builtin_pure.UnAlias(aliases, errfmt),

        builtin_e.TYPE: builtin_pure.Type(procs, aliases, exec_deps.search_path),
        builtin_e.HASH: builtin_pure.Hash(exec_deps.search_path),
        builtin_e.GETOPTS: builtin_pure.GetOpts(mem, errfmt),

        builtin_e.COLON: lambda arg_vec: 0,  # a "special" builtin 
        builtin_e.TRUE: lambda arg_vec: 0,
        builtin_e.FALSE: lambda arg_vec: 1,

        # Process
        builtin_e.WAIT: builtin_process.Wait(exec_deps.waiter,
                                             exec_deps.job_state, mem, errfmt),
        builtin_e.JOBS: builtin_process.Jobs(exec_deps.job_state),
        builtin_e.FG: builtin_process.Fg(exec_deps.job_state, exec_deps.waiter),
        builtin_e.BG: builtin_process.Bg(exec_deps.job_state),
        builtin_e.UMASK: builtin_process.Umask,

        # Oil
        builtin_e.REPR: builtin_oil.Repr(mem, errfmt),
        builtin_e.PUSH: builtin_oil.Push(mem, errfmt),
        builtin_e.USE: builtin_oil.Use(mem, errfmt),
    }

    ex = cmd_exec.Executor(mem, fd_state, procs, builtins, exec_opts,
                           parse_ctx, exec_deps)
    exec_deps.ex = ex

    word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
    exec_deps.word_ev = word_ev

    arith_ev = osh_expr_eval.ArithEvaluator(mem, exec_opts, word_ev, errfmt)
    exec_deps.arith_ev = arith_ev
    word_ev.arith_ev = arith_ev  # Another circular dependency

    bool_ev = osh_expr_eval.BoolEvaluator(mem, exec_opts, word_ev, errfmt)
    exec_deps.bool_ev = bool_ev

    expr_ev = expr_eval.OilEvaluator(mem, procs, ex, word_ev, errfmt)
    exec_deps.expr_ev = expr_ev

    tracer = dev.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
    exec_deps.tracer = tracer

    # HACK for circular deps
    ex.word_ev = word_ev
    ex.arith_ev = arith_ev
    ex.bool_ev = bool_ev
    ex.expr_ev = expr_ev
    ex.tracer = tracer

    word_ev.expr_ev = expr_ev

    spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
                                            comp_lookup)

    # Add some builtins that depend on the executor!
    complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
    builtins[builtin_e.COMPLETE] = complete_builtin
    builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)
    builtins[builtin_e.CD] = builtin.Cd(mem, dir_stack, ex, errfmt)
    builtins[builtin_e.JSON] = builtin_oil.Json(mem, ex, errfmt)

    sig_state = process.SignalState()
    sig_state.InitShell()

    builtins[builtin_e.TRAP] = builtin_process.Trap(sig_state, exec_deps.traps,
                                                    exec_deps.trap_nodes, ex,
                                                    errfmt)

    if lang == 'oil':
        # The Oil executor wraps an OSH executor?  It needs to be able to source
        # it.
        ex = oil_cmd_exec.OilExecutor(ex)

    # PromptEvaluator rendering is needed in non-interactive shells for @P.
    prompt_ev = prompt.Evaluator(lang, parse_ctx, ex, mem)
    exec_deps.prompt_ev = prompt_ev
    word_ev.prompt_ev = prompt_ev  # HACK for circular deps

    # History evaluation is a no-op if line_input is None.
    hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)

    if opts.c is not None:
        arena.PushSource(source.CFlag())
        line_reader = reader.StringLineReader(opts.c, arena)
        if opts.i:  # -c and -i can be combined
            exec_opts.interactive = True

    elif opts.i:  # force interactive
        arena.PushSource(source.Stdin(' -i'))
        line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
                                                   line_input, prompt_state,
                                                   sig_state)
        exec_opts.interactive = True

    else:
        try:
            script_name = arg_r.Peek()
        except IndexError:
            if sys.stdin.isatty():
                arena.PushSource(source.Interactive())
                line_reader = reader.InteractiveLineReader(
                    arena, prompt_ev, hist_ev, line_input, prompt_state,
                    sig_state)
                exec_opts.interactive = True
            else:
                arena.PushSource(source.Stdin(''))
                line_reader = reader.FileLineReader(sys.stdin, arena)
        else:
            arena.PushSource(source.MainFile(script_name))
            try:
                f = fd_state.Open(script_name)
            except OSError as e:
                ui.Stderr("osh: Couldn't open %r: %s", script_name,
                          posix.strerror(e.errno))
                return 1
            line_reader = reader.FileLineReader(f, arena)

    # TODO: assert arena.NumSourcePaths() == 1
    # TODO: .rc file needs its own arena.
    if lang == 'osh':
        c_parser = parse_ctx.MakeOshParser(line_reader)
    else:
        c_parser = parse_ctx.MakeOilParser(line_reader)

    if exec_opts.interactive:
        # Calculate ~/.config/oil/oshrc or oilrc
        # Use ~/.config/oil to avoid cluttering the user's home directory.  Some
        # users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.

        # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
        home_dir = process.GetHomeDir()
        assert home_dir is not None
        rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil',
                                              lang + 'rc')

        history_filename = os_path.join(home_dir, '.config/oil',
                                        'history_' + lang)

        if line_input:
            # NOTE: We're using a different WordEvaluator here.
            ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
                                                   arena)
            root_comp = completion.RootCompleter(ev, mem, comp_lookup,
                                                 compopt_state, comp_ui_state,
                                                 comp_ctx, debug_f)

            term_width = 0
            if opts.completion_display == 'nice':
                try:
                    term_width = libc.get_terminal_width()
                except IOError:  # stdin not a terminal
                    pass

            if term_width != 0:
                display = comp_ui.NiceDisplay(term_width, comp_ui_state,
                                              prompt_state, debug_f,
                                              line_input)
            else:
                display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                                 debug_f)

            _InitReadline(line_input, history_filename, root_comp, display,
                          debug_f)
            _InitDefaultCompletions(ex, complete_builtin, comp_lookup)

        else:  # Without readline module
            display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
                                             debug_f)

        sig_state.InitInteractiveShell(display)

        # NOTE: Call this AFTER _InitDefaultCompletions.
        try:
            SourceStartupFile(rc_path, lang, parse_ctx, ex)
        except util.UserExit as e:
            return e.status

        line_reader.Reset()  # After sourcing startup file, render $PS1

        prompt_plugin = prompt.UserPlugin(mem, parse_ctx, ex)
        try:
            status = main_loop.Interactive(opts, ex, c_parser, display,
                                           prompt_plugin, errfmt)
            if ex.MaybeRunExitTrap():
                status = ex.LastStatus()
        except util.UserExit as e:
            status = e.status
        return status

    nodes_out = [] if exec_opts.noexec else None

    if nodes_out is None and opts.parser_mem_dump:
        raise args.UsageError('--parser-mem-dump can only be used with -n')

    _tlog('Execute(node)')
    try:
        status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
        if ex.MaybeRunExitTrap():
            status = ex.LastStatus()
    except util.UserExit as e:
        status = e.status

    # Only print nodes if the whole parse succeeded.
    if nodes_out is not None and status == 0:
        if opts.parser_mem_dump:  # only valid in -n mode
            # This might be superstition, but we want to let the value stabilize
            # after parsing.  bash -c 'cat /proc/$$/status' gives different results
            # with a sleep.
            time.sleep(0.001)
            input_path = '/proc/%d/status' % posix.getpid()
            with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
                contents = f.read()
                f2.write(contents)
                log('Wrote %s to %s (--parser-mem-dump)', input_path,
                    opts.parser_mem_dump)

        ui.PrintAst(nodes_out, opts)

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if opts.runtime_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % posix.getpid()
        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
            contents = f.read()
            f2.write(contents)
            log('Wrote %s to %s (--runtime-mem-dump)', input_path,
                opts.runtime_mem_dump)

    # NOTE: We haven't closed the file opened with fd_state.Open
    return status