def __init__(self, dollar0, argv, environ, arena, has_main=False):
self.dollar0 = dollar0
self.argv_stack = [_ArgFrame(argv)]
self.var_stack = [{}]
# The debug_stack isn't strictly necessary for execution. We use it for
# crash dumps and for 3 parallel arrays: FUNCNAME, CALL_SOURCE,
# BASH_LINENO. The First frame points at the global vars and argv.
self.debug_stack = [(None, None, const.NO_INTEGER, 0, 0)]
self.bash_source = [] # for implementing BASH_SOURCE
self.has_main = has_main
if has_main:
self.bash_source.append(dollar0) # e.g. the filename
self.current_spid = const.NO_INTEGER
# Note: we're reusing these objects because they change on every single
# line! Don't want to allocate more than necsesary.
self.source_name = value.Str('')
self.line_num = value.Str('')
self.last_status = [0] # type: List[int] # a stack
self.pipe_status = [[]] # type: List[List[int]] # stack
self.last_job_id = -1 # Uninitialized value mutable public variable
# Done ONCE on initialization
self.root_pid = posix.getpid()
self._InitDefaults()
self._InitVarsFromEnv(environ)
self.arena = arena
def _CheckStatus(self, status, node):
"""Raises ErrExitFailure, maybe with location info attached."""
if self.exec_opts.ErrExit() and status != 0:
# NOTE: Sometimes location info is duplicated, like on UsageError, or a
# bad redirect. Also, pipelines can fail twice.
if node.tag == command_e.SimpleCommand:
reason = 'command in '
span_id = word.LeftMostSpanForWord(node.words[0])
elif node.tag == command_e.Assignment:
reason = 'assignment in '
span_id = self._SpanIdForAssignment(node)
elif node.tag == command_e.Subshell:
reason = 'subshell invoked from '
span_id = node.spids[0]
elif node.tag == command_e.Pipeline:
# The whole pipeline can fail separately
reason = 'pipeline invoked from '
span_id = node.spids[0] # only one spid
else:
# NOTE: The fallback of CurrentSpanId() fills this in.
reason = ''
span_id = const.NO_INTEGER
raise util.ErrExitFailure(
'Exiting with status %d (%sPID %d)', status, reason, posix.getpid(),
span_id=span_id, status=status)
def testRead(self):
if posix_.environ.get('EINTR_TEST'):
# Now we can do kill -TERM PID can get EINTR.
# Or Ctrl-C for KeyboardInterrupt
signal.signal(signal.SIGTERM, _Handler)
log('Hanging on read in pid %d', posix_.getpid())
posix_.read(0, 1)
def testWrite(self):
if posix_.environ.get('EINTR_TEST'):
signal.signal(signal.SIGTERM, _Handler)
r, w = posix_.pipe()
log('Hanging on write in pid %d', posix_.getpid())
# 1 byte bigger than pipe size
n = posix_.write(w, 'x' * 65537)
log('1: Wrote %d bytes', n)
# write returns early when a signal interrupts it, and we read at least
# one byte! We do NOT get EINTR>
# On the second try, it didn't write anything, and we get EINTR!
log('Second try (pid %d)', posix_.getpid())
n = posix_.write(w, 'x' * 65537)
log('2: Wrote %d bytes', n)
def testWaitpid(self):
if posix_.environ.get('EINTR_TEST'):
# Now we can do kill -TERM PID can get EINTR.
signal.signal(signal.SIGTERM, _Handler)
p = subprocess.Popen(['sleep', '5'])
log('started sleep pid %d', p.pid)
log('Hanging on waitpid in pid %d', posix_.getpid())
posix_.waitpid(-1, 0)
def testPrint(self):
# Conclusion: print CAN raise IOError with EINTR.
if posix_.environ.get('EINTR_TEST'):
signal.signal(signal.SIGTERM, _Handler)
r, w = posix_.pipe()
log('Hanging on write in pid %d', posix_.getpid())
f = posix_.fdopen(w, 'w')
# 1 byte bigger than pipe size
print('x' * 65537, file=f)
log('1: done')
# write returns early when a signal interrupts it, and we read at least
# one byte! We do NOT get EINTR>
# On the second try, it didn't write anything, and we get EINTR!
log('Second try (pid %d)', posix_.getpid())
print('x' * 65537, file=f)
log('2: done')
def MaybeDump(self, status):
# type: (int) -> None
"""Write the dump as JSON.
User can configure it two ways:
- dump unconditionally -- a daily cron job. This would be fine.
- dump on non-zero exit code
OIL_FAIL
Maybe counters are different than failure
OIL_CRASH_DUMP='function alias trap completion stack' ?
OIL_COUNTER_DUMP='function alias trap completion'
and then
I think both of these should dump the (path, mtime, checksum) of the source
they ran? And then you can match those up with source control or whatever?
"""
if not self.collected:
return
if mylib.PYTHON: # can't translate due to open()
my_pid = posix.getpid() # Get fresh PID here
# Other things we need: the reason for the crash! _ErrorWithLocation is
# required I think.
d = {
'var_stack': self.var_stack,
'argv_stack': self.argv_stack,
'debug_stack': self.debug_stack,
'error': self.error,
'status': status,
'pid': my_pid,
}
# TODO: Add PID here
path = os_path.join(self.crash_dump_dir,
'%d-osh-crash-dump.json' % my_pid)
with open(path, 'w') as f:
import json
json.dump(d, f, indent=2)
#print(repr(d), file=f)
log('[%d] Wrote crash dump to %s', my_pid, path)
def Start(self, why):
# type: (trace_t) -> int
"""Start this process with fork(), handling redirects."""
# TODO: If OSH were a job control shell, we might need to call some of
# these here. They control the distribution of signals, some of which
# originate from a terminal. All the processes in a pipeline should be in
# a single process group.
#
# - posix.setpgid()
# - posix.setpgrp()
# - posix.tcsetpgrp()
#
# NOTE: posix.setsid() isn't called by the shell; it's should be called by the
# login program that starts the shell.
#
# The whole job control mechanism is complicated and hacky.
pid = posix.fork()
if pid < 0:
# When does this happen?
raise RuntimeError('Fatal error in posix.fork()')
elif pid == 0: # child
pyos.SignalState_AfterForkingChild()
for st in self.state_changes:
st.Apply()
self.tracer.SetProcess(posix.getpid())
self.thunk.Run()
# Never returns
#log('STARTED process %s, pid = %d', self, pid)
self.tracer.OnProcessStart(pid, why)
# Class invariant: after the process is started, it stores its PID.
self.pid = pid
# Program invariant: We keep track of every child process!
self.job_state.AddChildProcess(pid, self)
return pid
def _tlog(msg):
# type: (str) -> None
pid = posix.getpid() # TODO: Maybe remove PID later.
print('[%d] %.3f %s' % (pid, (time.time() - start_time) * 1000, msg))
def Main(lang, arg_r, environ, login_shell, loader, line_input):
# type: (str, args.Reader, Dict[str, str], bool, pyutil._ResourceLoader, Any) -> int
"""The full shell lifecycle. Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, arg_r: command line arguments
environ: environment
login_shell: Was - on the front?
loader: to get help, version, grammar, etc.
line_input: optional GNU readline
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - shopt -s oil:all
# - Change the prompt in the interactive shell?
# osh-pure:
# - no oil grammar
# - no expression evaluator
# - no interactive shell, or line_input
# - no process.*
# process.{ExternalProgram,Waiter,FdState,JobState,SignalState} -- we want
# to evaluate config files without any of these
# Modules not translated yet: completion, comp_ui, builtin_comp, process
# - word evaluator
# - shouldn't glob? set -o noglob? or hard failure?
# - ~ shouldn't read from the file system
# - I guess it can just be the HOME=HOME?
# Builtin:
# shellvm -c 'echo hi'
# shellvm <<< 'echo hi'
argv0 = arg_r.Peek()
assert argv0 is not None
arg_r.Next()
assert lang in ('osh', 'oil'), lang
try:
attrs = flag_spec.ParseMore('main', arg_r)
except error.Usage as e:
stderr_line('osh usage error: %s', e.msg)
return 2
flag = arg_types.main(attrs.attrs)
arena = alloc.Arena()
errfmt = ui.ErrorFormatter(arena)
help_builtin = builtin_misc.Help(loader, errfmt)
if flag.help:
help_builtin.Run(pure.MakeBuiltinArgv(['%s-usage' % lang]))
return 0
if flag.version:
# OSH version is the only binary in Oil right now, so it's all one version.
pyutil.ShowAppVersion('Oil', loader)
return 0
no_str = None # type: str
debug_stack = [] # type: List[state.DebugFrame]
if arg_r.AtEnd():
dollar0 = argv0
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
# Copy quirky bash behavior.
frame0 = state.DebugFrame(dollar0, 'main', no_str, state.LINE_ZERO, 0,
0)
debug_stack.append(frame0)
# Copy quirky bash behavior.
frame1 = state.DebugFrame(no_str, no_str, no_str, runtime.NO_SPID, 0, 0)
debug_stack.append(frame1)
script_name = arg_r.Peek() # type: Optional[str]
arg_r.Next()
mem = state.Mem(dollar0, arg_r.Rest(), arena, debug_stack)
version_str = pyutil.GetVersion(loader)
state.InitMem(mem, environ, version_str)
builtin_funcs.Init(mem)
procs = {} # type: Dict[str, command__ShFunction]
job_state = process.JobState()
fd_state = process.FdState(errfmt, job_state, mem)
opt_hook = ShellOptHook(line_input)
parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
# TODO: only MutableOpts needs mem, so it's not a true circular dep.
mem.exec_opts = exec_opts # circular dep
if attrs.show_options: # special case: sh -o
mutable_opts.ShowOptions([])
return 0
# Set these BEFORE processing flags, so they can be overridden.
if lang == 'oil':
mutable_opts.SetShoptOption('oil:all', True)
builtin_pure.SetShellOpts(mutable_opts, attrs.opt_changes,
attrs.shopt_changes)
# feedback between runtime and parser
aliases = {} # type: Dict[str, str]
oil_grammar = pyutil.LoadOilGrammar(loader)
if flag.one_pass_parse and not exec_opts.noexec():
raise error.Usage('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
parse_ctx.Init_OnePassParse(flag.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena, parse_opts, aliases,
oil_grammar)
comp_ctx.Init_Trail(trail1)
comp_ctx.Init_OnePassParse(True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
oil_grammar)
hist_ctx.Init_Trail(trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - cmd_ev and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - cmd_ev and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
cmd_deps = cmd_eval.Deps()
cmd_deps.mutable_opts = mutable_opts
# TODO: In general, cmd_deps are shared between the mutually recursive
# evaluators. Some of the four below are only shared between a builtin and
# the CommandEvaluator, so we could put them somewhere else.
cmd_deps.traps = {}
cmd_deps.trap_nodes = [] # TODO: Clear on fork() to avoid duplicates
waiter = process.Waiter(job_state, exec_opts)
my_pid = posix.getpid()
debug_path = ''
debug_dir = environ.get('OSH_DEBUG_DIR')
if flag.debug_file is not None:
# --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = flag.debug_file
elif debug_dir is not None:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if len(debug_path):
# This will be created as an empty file if it doesn't exist, or it could be
# a pipe.
try:
debug_f = util.DebugFile(
fd_state.OpenForWrite(debug_path)) # type: util._DebugFile
except OSError as e:
stderr_line("osh: Couldn't open %r: %s", debug_path,
posix.strerror(e.errno))
return 2
else:
debug_f = util.NullDebugFile()
cmd_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
if mylib.PYTHON:
iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid,
arg_r.argv)
if len(debug_path):
debug_f.log('Writing logs to %r', debug_path)
interp = environ.get('OSH_HIJACK_SHEBANG', '')
search_path = state.SearchPath(mem)
ext_prog = process.ExternalProgram(interp, fd_state, errfmt, debug_f)
splitter = split.SplitContext(mem)
# split() builtin
# TODO: Accept IFS as a named arg? split('a b', IFS=' ')
builtin_funcs.SetGlobalFunc(
mem,
'split',
lambda s, ifs=None: splitter.SplitForWordEval(s, ifs=ifs))
# glob() builtin
# TODO: This is instantiation is duplicated in osh/word_eval.py
globber = glob_.Globber(exec_opts)
builtin_funcs.SetGlobalFunc(mem, 'glob', lambda s: globber.OilFuncCall(s))
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = environ.get('OSH_CRASH_DUMP_DIR', '')
cmd_deps.dumper = dev.CrashDumper(crash_dump_dir)
if flag.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(mylib.Stderr())
comp_lookup = completion.Lookup()
# Various Global State objects to work around readline interfaces
compopt_state = completion.OptionState()
comp_ui_state = comp_ui.State()
prompt_state = comp_ui.PromptState()
dir_stack = state.DirStack()
#
# Initialize builtins that don't depend on evaluators
#
builtins = {} # type: Dict[int, vm._Builtin]
pure.AddPure(builtins, mem, procs, mutable_opts, aliases, search_path,
errfmt)
pure.AddIO(builtins, mem, dir_stack, exec_opts, splitter, errfmt)
AddProcess(builtins, mem, ext_prog, fd_state, job_state, waiter,
search_path, errfmt)
AddOil(builtins, mem, errfmt)
builtins[builtin_i.help] = help_builtin
# Interactive, depend on line_input
builtins[builtin_i.bind] = builtin_lib.Bind(line_input, errfmt)
builtins[builtin_i.history] = builtin_lib.History(line_input,
mylib.Stdout())
#
# Assignment builtins
#
assign_b = {} # type: Dict[int, vm._AssignBuiltin]
new_var = builtin_assign.NewVar(mem, procs, errfmt)
assign_b[builtin_i.declare] = new_var
assign_b[builtin_i.typeset] = new_var
assign_b[builtin_i.local] = new_var
assign_b[builtin_i.export_] = builtin_assign.Export(mem, errfmt)
assign_b[builtin_i.readonly] = builtin_assign.Readonly(mem, errfmt)
#
# Initialize Evaluators
#
arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, parse_ctx, errfmt)
bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, parse_ctx, errfmt)
expr_ev = expr_eval.OilEvaluator(mem, procs, splitter, errfmt)
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)
cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
arena, cmd_deps)
shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
builtins, search_path, ext_prog, waiter,
job_state, fd_state, errfmt)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, mem)
tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
trace_f)
# Wire up circular dependencies.
vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
prompt_ev, tracer)
#
# Initialize builtins that depend on evaluators
#
# note: 'printf -v a[i]' and 'unset a[i]' require same deps
builtins[builtin_i.printf] = builtin_printf.Printf(mem, exec_opts,
parse_ctx, arith_ev,
errfmt)
builtins[builtin_i.unset] = builtin_assign.Unset(mem, exec_opts, procs,
parse_ctx, arith_ev,
errfmt)
builtins[builtin_i.eval] = builtin_meta.Eval(parse_ctx, exec_opts, cmd_ev)
source_builtin = builtin_meta.Source(parse_ctx, search_path, cmd_ev,
fd_state, errfmt)
builtins[builtin_i.source] = source_builtin
builtins[builtin_i.dot] = source_builtin
builtins[builtin_i.builtin] = builtin_meta.Builtin(shell_ex, errfmt)
builtins[builtin_i.command] = builtin_meta.Command(shell_ex, procs,
aliases, search_path)
spec_builder = builtin_comp.SpecBuilder(cmd_ev, parse_ctx, word_ev,
splitter, comp_lookup)
complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
builtins[builtin_i.complete] = complete_builtin
builtins[builtin_i.compgen] = builtin_comp.CompGen(spec_builder)
builtins[builtin_i.compopt] = builtin_comp.CompOpt(compopt_state, errfmt)
builtins[builtin_i.compadjust] = builtin_comp.CompAdjust(mem)
# These builtins take blocks, and thus need cmd_ev.
builtins[builtin_i.cd] = builtin_misc.Cd(mem, dir_stack, cmd_ev, errfmt)
builtins[builtin_i.json] = builtin_oil.Json(mem, cmd_ev, errfmt)
sig_state = pyos.SignalState()
sig_state.InitShell()
builtins[builtin_i.trap] = builtin_process.Trap(sig_state, cmd_deps.traps,
cmd_deps.trap_nodes,
parse_ctx, errfmt)
# History evaluation is a no-op if line_input is None.
hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)
if flag.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(flag.c,
arena) # type: reader._Reader
if flag.i: # -c and -i can be combined
mutable_opts.set_interactive()
elif flag.i: # force interactive
arena.PushSource(source.Stdin(' -i'))
line_reader = py_reader.InteractiveLineReader(arena, prompt_ev,
hist_ev, line_input,
prompt_state)
mutable_opts.set_interactive()
else:
if script_name is None:
stdin = mylib.Stdin()
if stdin.isatty():
arena.PushSource(source.Interactive())
line_reader = py_reader.InteractiveLineReader(
arena, prompt_ev, hist_ev, line_input, prompt_state)
mutable_opts.set_interactive()
else:
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
except OSError as e:
stderr_line("osh: Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
c_parser = parse_ctx.MakeOshParser(line_reader)
if exec_opts.interactive():
# bash: 'set -o emacs' is the default only in the interactive shell
mutable_opts.set_emacs()
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = pyos.GetMyHomeDir()
assert home_dir is not None
history_filename = os_path.join(home_dir,
'.config/oil/history_%s' % lang)
if line_input:
# NOTE: We're using a different WordEvaluator here.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
errfmt)
ev.arith_ev = arith_ev
ev.expr_ev = expr_ev
ev.prompt_ev = prompt_ev
ev.CheckCircularDeps()
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
compopt_state, comp_ui_state,
comp_ctx, debug_f)
term_width = 0
if flag.completion_display == 'nice':
try:
term_width = libc.get_terminal_width()
except IOError: # stdin not a terminal
pass
if term_width != 0:
display = comp_ui.NiceDisplay(
term_width, comp_ui_state, prompt_state, debug_f,
line_input) # type: comp_ui._IDisplay
else:
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
comp_ui.InitReadline(line_input, history_filename, root_comp,
display, debug_f)
_InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup)
else: # Without readline module
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
sig_state.InitInteractiveShell(display)
rc_path = flag.rcfile or os_path.join(home_dir,
'.config/oil/%src' % lang)
try:
# NOTE: Should be called AFTER _InitDefaultCompletions.
SourceStartupFile(fd_state, rc_path, lang, parse_ctx, cmd_ev)
except util.UserExit as e:
return e.status
line_reader.Reset() # After sourcing startup file, render $PS1
prompt_plugin = prompt.UserPlugin(mem, parse_ctx, cmd_ev)
try:
status = main_loop.Interactive(flag, cmd_ev, c_parser, display,
prompt_plugin, errfmt)
except util.UserExit as e:
status = e.status
box = [status]
cmd_ev.MaybeRunExitTrap(box)
status = box[0]
return status
if flag.rcfile: # bash doesn't have this warning, but it's useful
stderr_line('osh warning: --rcfile ignored in non-interactive shell')
if exec_opts.noexec():
status = 0
try:
node = main_loop.ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
status = 2
if status == 0:
if flag.parser_mem_dump is not None: # only valid in -n mode
input_path = '/proc/%d/status' % posix.getpid()
pyutil.CopyFile(input_path, flag.parser_mem_dump)
ui.PrintAst(node, flag)
else:
if flag.parser_mem_dump is not None:
raise error.Usage('--parser-mem-dump can only be used with -n')
try:
status = main_loop.Batch(cmd_ev,
c_parser,
arena,
cmd_flags=cmd_eval.IsMainProgram)
except util.UserExit as e:
status = e.status
box = [status]
cmd_ev.MaybeRunExitTrap(box)
status = box[0]
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if flag.runtime_mem_dump is not None:
input_path = '/proc/%d/status' % posix.getpid()
pyutil.CopyFile(input_path, flag.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def ShellMain(lang, argv0, argv, login_shell):
"""Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox.
login_shell: Was - on the front?
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - the parser and executor
# - Change the prompt in the interactive shell?
assert lang in ('osh', 'oil'), lang
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.Stderr('osh usage error: %s', e.msg)
return 2
# NOTE: This has a side effect of deleting _OVM_* from the environment!
# TODO: Thread this throughout the program, and get rid of the global
# variable in core/util.py. Rename to InitResourceLaoder(). It's now only
# used for the 'help' builtin and --version.
loader = pyutil.GetResourceLoader()
if opts.help:
builtin.Help(['%s-usage' % lang], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
arena = alloc.Arena()
errfmt = ui.ErrorFormatter(arena)
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
builtin_funcs.Init(mem)
procs = {}
job_state = process.JobState()
fd_state = process.FdState(errfmt, job_state)
parse_opts = parse_lib.OilParseOptions()
exec_opts = state.ExecOpts(mem, parse_opts, line_input)
if opts.show_options: # special case: sh -o
exec_opts.ShowOptions([])
return 0
builtin_pure.SetExecOpts(exec_opts, opts.opt_changes, opts.shopt_changes)
aliases = {} # feedback between runtime and parser
oil_grammar = meta.LoadOilGrammar(loader)
if opts.one_pass_parse and not exec_opts.noexec:
raise args.UsageError('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena,
parse_opts,
aliases,
oil_grammar,
one_pass_parse=opts.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena,
parse_opts,
aliases,
oil_grammar,
trail=trail1,
one_pass_parse=True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena,
parse_opts,
aliases,
oil_grammar,
trail=trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - ex and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - ex and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
exec_deps = cmd_exec.Deps()
# TODO: In general, exec_deps are shared between the mutually recursive
# evaluators. Some of the four below are only shared between a builtin and
# the Executor, so we could put them somewhere else.
exec_deps.traps = {}
exec_deps.trap_nodes = [] # TODO: Clear on fork() to avoid duplicates
exec_deps.job_state = job_state
# note: exec_opts.interactive set later
exec_deps.waiter = process.Waiter(job_state, exec_opts)
exec_deps.errfmt = errfmt
my_pid = posix.getpid()
debug_path = ''
debug_dir = posix.environ.get('OSH_DEBUG_DIR')
if opts.debug_file: # --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = opts.debug_file
elif debug_dir:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if debug_path:
# This will be created as an empty file if it doesn't exist, or it could be
# a pipe.
try:
debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
except OSError as e:
ui.Stderr("osh: Couldn't open %r: %s", debug_path,
posix.strerror(e.errno))
return 2
else:
debug_f = util.NullDebugFile()
exec_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
if debug_path:
debug_f.log('Writing logs to %r', debug_path)
interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
exec_deps.search_path = state.SearchPath(mem)
exec_deps.ext_prog = process.ExternalProgram(interp, fd_state,
exec_deps.search_path, errfmt,
debug_f)
splitter = split.SplitContext(mem)
exec_deps.splitter = splitter
# split() builtin
builtin_funcs.SetGlobalFunc(mem, 'split',
lambda s: splitter.SplitForWordEval(s))
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
exec_deps.dumper = dev.CrashDumper(crash_dump_dir)
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
exec_deps.trace_f = trace_f
comp_lookup = completion.Lookup()
# Various Global State objects to work around readline interfaces
compopt_state = completion.OptionState()
comp_ui_state = comp_ui.State()
prompt_state = comp_ui.PromptState()
dir_stack = state.DirStack()
new_var = builtin_assign.NewVar(mem, procs, errfmt)
builtins = { # Lookup
builtin_e.ECHO: builtin_pure.Echo(exec_opts),
builtin_e.PRINTF: builtin_printf.Printf(mem, parse_ctx, errfmt),
builtin_e.PUSHD: builtin.Pushd(mem, dir_stack, errfmt),
builtin_e.POPD: builtin.Popd(mem, dir_stack, errfmt),
builtin_e.DIRS: builtin.Dirs(mem, dir_stack, errfmt),
builtin_e.PWD: builtin.Pwd(mem, errfmt),
builtin_e.READ: builtin.Read(splitter, mem),
builtin_e.HELP: builtin.Help(loader, errfmt),
builtin_e.HISTORY: builtin.History(line_input),
# Completion (more added below)
builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state, errfmt),
builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
# test / [ differ by need_right_bracket
builtin_e.TEST: builtin_bracket.Test(False, errfmt),
builtin_e.BRACKET: builtin_bracket.Test(True, errfmt),
# Assignment (which are pure)
builtin_e.DECLARE: new_var,
builtin_e.TYPESET: new_var,
builtin_e.LOCAL: new_var,
builtin_e.EXPORT: builtin_assign.Export(mem, errfmt),
builtin_e.READONLY: builtin_assign.Readonly(mem, errfmt),
builtin_e.UNSET: builtin_assign.Unset(mem, procs, errfmt),
builtin_e.SHIFT: builtin_assign.Shift(mem),
# Pure
builtin_e.SET: builtin_pure.Set(exec_opts, mem),
builtin_e.SHOPT: builtin_pure.Shopt(exec_opts),
builtin_e.ALIAS: builtin_pure.Alias(aliases, errfmt),
builtin_e.UNALIAS: builtin_pure.UnAlias(aliases, errfmt),
builtin_e.TYPE: builtin_pure.Type(procs, aliases, exec_deps.search_path),
builtin_e.HASH: builtin_pure.Hash(exec_deps.search_path),
builtin_e.GETOPTS: builtin_pure.GetOpts(mem, errfmt),
builtin_e.COLON: lambda arg_vec: 0, # a "special" builtin
builtin_e.TRUE: lambda arg_vec: 0,
builtin_e.FALSE: lambda arg_vec: 1,
# Process
builtin_e.WAIT: builtin_process.Wait(exec_deps.waiter,
exec_deps.job_state, mem, errfmt),
builtin_e.JOBS: builtin_process.Jobs(exec_deps.job_state),
builtin_e.FG: builtin_process.Fg(exec_deps.job_state, exec_deps.waiter),
builtin_e.BG: builtin_process.Bg(exec_deps.job_state),
builtin_e.UMASK: builtin_process.Umask,
# Oil
builtin_e.REPR: builtin_oil.Repr(mem, errfmt),
builtin_e.PUSH: builtin_oil.Push(mem, errfmt),
builtin_e.USE: builtin_oil.Use(mem, errfmt),
}
ex = cmd_exec.Executor(mem, fd_state, procs, builtins, exec_opts,
parse_ctx, exec_deps)
exec_deps.ex = ex
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
exec_deps.word_ev = word_ev
arith_ev = osh_expr_eval.ArithEvaluator(mem, exec_opts, word_ev, errfmt)
exec_deps.arith_ev = arith_ev
word_ev.arith_ev = arith_ev # Another circular dependency
bool_ev = osh_expr_eval.BoolEvaluator(mem, exec_opts, word_ev, errfmt)
exec_deps.bool_ev = bool_ev
expr_ev = expr_eval.OilEvaluator(mem, procs, ex, word_ev, errfmt)
exec_deps.expr_ev = expr_ev
tracer = dev.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
exec_deps.tracer = tracer
# HACK for circular deps
ex.word_ev = word_ev
ex.arith_ev = arith_ev
ex.bool_ev = bool_ev
ex.expr_ev = expr_ev
ex.tracer = tracer
word_ev.expr_ev = expr_ev
spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
comp_lookup)
# Add some builtins that depend on the executor!
complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
builtins[builtin_e.COMPLETE] = complete_builtin
builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)
builtins[builtin_e.CD] = builtin.Cd(mem, dir_stack, ex, errfmt)
builtins[builtin_e.JSON] = builtin_oil.Json(mem, ex, errfmt)
sig_state = process.SignalState()
sig_state.InitShell()
builtins[builtin_e.TRAP] = builtin_process.Trap(sig_state, exec_deps.traps,
exec_deps.trap_nodes, ex,
errfmt)
if lang == 'oil':
# The Oil executor wraps an OSH executor? It needs to be able to source
# it.
ex = oil_cmd_exec.OilExecutor(ex)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, ex, mem)
exec_deps.prompt_ev = prompt_ev
word_ev.prompt_ev = prompt_ev # HACK for circular deps
# History evaluation is a no-op if line_input is None.
hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)
if opts.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource(source.Stdin(' -i'))
line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
line_input, prompt_state,
sig_state)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource(source.Interactive())
line_reader = reader.InteractiveLineReader(
arena, prompt_ev, hist_ev, line_input, prompt_state,
sig_state)
exec_opts.interactive = True
else:
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
except OSError as e:
ui.Stderr("osh: Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
if lang == 'osh':
c_parser = parse_ctx.MakeOshParser(line_reader)
else:
c_parser = parse_ctx.MakeOilParser(line_reader)
if exec_opts.interactive:
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = process.GetHomeDir()
assert home_dir is not None
rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil',
lang + 'rc')
history_filename = os_path.join(home_dir, '.config/oil',
'history_' + lang)
if line_input:
# NOTE: We're using a different WordEvaluator here.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
arena)
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
compopt_state, comp_ui_state,
comp_ctx, debug_f)
term_width = 0
if opts.completion_display == 'nice':
try:
term_width = libc.get_terminal_width()
except IOError: # stdin not a terminal
pass
if term_width != 0:
display = comp_ui.NiceDisplay(term_width, comp_ui_state,
prompt_state, debug_f,
line_input)
else:
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
_InitReadline(line_input, history_filename, root_comp, display,
debug_f)
_InitDefaultCompletions(ex, complete_builtin, comp_lookup)
else: # Without readline module
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
sig_state.InitInteractiveShell(display)
# NOTE: Call this AFTER _InitDefaultCompletions.
try:
SourceStartupFile(rc_path, lang, parse_ctx, ex)
except util.UserExit as e:
return e.status
line_reader.Reset() # After sourcing startup file, render $PS1
prompt_plugin = prompt.UserPlugin(mem, parse_ctx, ex)
try:
status = main_loop.Interactive(opts, ex, c_parser, display,
prompt_plugin, errfmt)
if ex.MaybeRunExitTrap():
status = ex.LastStatus()
except util.UserExit as e:
status = e.status
return status
nodes_out = [] if exec_opts.noexec else None
if nodes_out is None and opts.parser_mem_dump:
raise args.UsageError('--parser-mem-dump can only be used with -n')
_tlog('Execute(node)')
try:
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
if ex.MaybeRunExitTrap():
status = ex.LastStatus()
except util.UserExit as e:
status = e.status
# Only print nodes if the whole parse succeeded.
if nodes_out is not None and status == 0:
if opts.parser_mem_dump: # only valid in -n mode
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def ShowFdState(): # type: () -> None import subprocess import posix_ as posix subprocess.call(['ls', '-l', '/proc/%d/fd' % posix.getpid()])
def ShellMain(lang, argv0, argv, login_shell):
# type: (str, str, List[str], bool) -> int
"""Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox.
login_shell: Was - on the front?
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - the parser and executor
# - Change the prompt in the interactive shell?
assert lang in ('osh', 'oil'), lang
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.Stderr('osh usage error: %s', e.msg)
return 2
# NOTE: This has a side effect of deleting _OVM_* from the environment!
# TODO: Thread this throughout the program, and get rid of the global
# variable in core/util.py. Rename to InitResourceLaoder(). It's now only
# used for the 'help' builtin and --version.
loader = pyutil.GetResourceLoader()
if opts.help:
builtin_misc.Help(['%s-usage' % lang], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
no_str = None # type: str
debug_stack = []
if arg_r.AtEnd():
dollar0 = argv0
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
# Copy quirky bash behavior.
frame0 = state.DebugFrame(dollar0, 'main', no_str, state.LINE_ZERO, 0,
0)
debug_stack.append(frame0)
# Copy quirky bash behavior.
frame1 = state.DebugFrame(no_str, no_str, no_str, runtime.NO_SPID, 0, 0)
debug_stack.append(frame1)
arena = alloc.Arena()
errfmt = ui.ErrorFormatter(arena)
mem = state.Mem(dollar0, argv[arg_r.i + 1:], arena, debug_stack)
state.InitMem(mem, posix.environ)
builtin_funcs.Init(mem)
procs = {}
job_state = process.JobState()
fd_state = process.FdState(errfmt, job_state)
opt_hook = ShellOptHook(line_input)
parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
# TODO: only MutableOpts needs mem, so it's not a true circular dep.
mem.exec_opts = exec_opts # circular dep
if opts.show_options: # special case: sh -o
mutable_opts.ShowOptions([])
return 0
# Set these BEFORE processing flags, so they can be overridden.
if lang == 'oil':
mutable_opts.SetShoptOption('oil:all', True)
builtin_pure.SetShellOpts(mutable_opts, opts.opt_changes,
opts.shopt_changes)
aliases = {} # feedback between runtime and parser
oil_grammar = meta.LoadOilGrammar(loader)
if opts.one_pass_parse and not exec_opts.noexec():
raise args.UsageError('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
parse_ctx.Init_OnePassParse(opts.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena, parse_opts, aliases,
oil_grammar)
comp_ctx.Init_Trail(trail1)
comp_ctx.Init_OnePassParse(True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
oil_grammar)
hist_ctx.Init_Trail(trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - ex and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - ex and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
exec_deps = cmd_exec.Deps()
exec_deps.mutable_opts = mutable_opts
# TODO: In general, exec_deps are shared between the mutually recursive
# evaluators. Some of the four below are only shared between a builtin and
# the Executor, so we could put them somewhere else.
exec_deps.traps = {}
exec_deps.trap_nodes = [] # TODO: Clear on fork() to avoid duplicates
exec_deps.job_state = job_state
exec_deps.waiter = process.Waiter(job_state, exec_opts)
exec_deps.errfmt = errfmt
my_pid = posix.getpid()
debug_path = ''
debug_dir = posix.environ.get('OSH_DEBUG_DIR')
if opts.debug_file: # --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = opts.debug_file
elif debug_dir:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if debug_path:
# This will be created as an empty file if it doesn't exist, or it could be
# a pipe.
try:
debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
except OSError as e:
ui.Stderr("osh: Couldn't open %r: %s", debug_path,
posix.strerror(e.errno))
return 2
else:
debug_f = util.NullDebugFile()
exec_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
if debug_path:
debug_f.log('Writing logs to %r', debug_path)
interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
exec_deps.search_path = state.SearchPath(mem)
exec_deps.ext_prog = process.ExternalProgram(interp, fd_state,
exec_deps.search_path, errfmt,
debug_f)
splitter = split.SplitContext(mem)
# split() builtin
# TODO: Accept IFS as a named arg? split('a b', IFS=' ')
builtin_funcs.SetGlobalFunc(
mem,
'split',
lambda s, ifs=None: splitter.SplitForWordEval(s, ifs=ifs))
# glob() builtin
# TODO: This is instantiation is duplicated in osh/word_eval.py
globber = glob_.Globber(exec_opts)
builtin_funcs.SetGlobalFunc(mem, 'glob', lambda s: globber.OilFuncCall(s))
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
exec_deps.dumper = dev.CrashDumper(crash_dump_dir)
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
exec_deps.trace_f = trace_f
comp_lookup = completion.Lookup()
# Various Global State objects to work around readline interfaces
compopt_state = completion.OptionState()
comp_ui_state = comp_ui.State()
prompt_state = comp_ui.PromptState()
dir_stack = state.DirStack()
new_var = builtin_assign.NewVar(mem, procs, errfmt)
builtins = { # Lookup
builtin_i.echo: builtin_pure.Echo(exec_opts),
builtin_i.printf: builtin_printf.Printf(mem, parse_ctx, errfmt),
builtin_i.pushd: builtin_misc.Pushd(mem, dir_stack, errfmt),
builtin_i.popd: builtin_misc.Popd(mem, dir_stack, errfmt),
builtin_i.dirs: builtin_misc.Dirs(mem, dir_stack, errfmt),
builtin_i.pwd: builtin_misc.Pwd(mem, errfmt),
builtin_i.times: builtin_misc.Times(),
builtin_i.read: builtin_misc.Read(splitter, mem),
builtin_i.help: builtin_misc.Help(loader, errfmt),
builtin_i.history: builtin_misc.History(line_input),
# Completion (more added below)
builtin_i.compopt: builtin_comp.CompOpt(compopt_state, errfmt),
builtin_i.compadjust: builtin_comp.CompAdjust(mem),
# test / [ differ by need_right_bracket
builtin_i.test: builtin_bracket.Test(False, exec_opts, errfmt),
builtin_i.bracket: builtin_bracket.Test(True, exec_opts, errfmt),
# ShAssignment (which are pure)
builtin_i.declare: new_var,
builtin_i.typeset: new_var,
builtin_i.local: new_var,
builtin_i.export_: builtin_assign.Export(mem, errfmt),
builtin_i.readonly: builtin_assign.Readonly(mem, errfmt),
builtin_i.unset: builtin_assign.Unset(mem, procs, errfmt),
builtin_i.shift: builtin_assign.Shift(mem),
# Pure
builtin_i.set: builtin_pure.Set(mutable_opts, mem),
builtin_i.shopt: builtin_pure.Shopt(mutable_opts),
builtin_i.alias: builtin_pure.Alias(aliases, errfmt),
builtin_i.unalias: builtin_pure.UnAlias(aliases, errfmt),
builtin_i.type: builtin_pure.Type(procs, aliases, exec_deps.search_path),
builtin_i.hash: builtin_pure.Hash(exec_deps.search_path),
builtin_i.getopts: builtin_pure.GetOpts(mem, errfmt),
builtin_i.colon: builtin_pure.Boolean(0), # a "special" builtin
builtin_i.true_: builtin_pure.Boolean(0),
builtin_i.false_: builtin_pure.Boolean(1),
# Process
builtin_i.wait: builtin_process.Wait(exec_deps.waiter,
exec_deps.job_state, mem, errfmt),
builtin_i.jobs: builtin_process.Jobs(exec_deps.job_state),
builtin_i.fg: builtin_process.Fg(exec_deps.job_state, exec_deps.waiter),
builtin_i.bg: builtin_process.Bg(exec_deps.job_state),
builtin_i.umask: builtin_process.Umask(),
# Oil
builtin_i.push: builtin_oil.Push(mem, errfmt),
builtin_i.append: builtin_oil.Append(mem, errfmt),
builtin_i.write: builtin_oil.Write(mem, errfmt),
builtin_i.getline: builtin_oil.Getline(mem, errfmt),
builtin_i.repr: builtin_oil.Repr(mem, errfmt),
builtin_i.use: builtin_oil.Use(mem, errfmt),
builtin_i.opts: builtin_oil.Opts(mem, errfmt),
}
arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, errfmt)
bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, errfmt)
expr_ev = expr_eval.OilEvaluator(mem, procs, errfmt)
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)
ex = cmd_exec.Executor(mem, fd_state, procs, builtins, exec_opts,
parse_ctx, exec_deps)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, mem)
tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
trace_f)
# Wire up circular dependencies.
vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, ex, prompt_ev,
tracer)
spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
comp_lookup)
# Add some builtins that depend on the executor!
complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
builtins[builtin_i.complete] = complete_builtin
builtins[builtin_i.compgen] = builtin_comp.CompGen(spec_builder)
builtins[builtin_i.cd] = builtin_misc.Cd(mem, dir_stack, ex, errfmt)
builtins[builtin_i.json] = builtin_oil.Json(mem, ex, errfmt)
sig_state = process.SignalState()
sig_state.InitShell()
builtins[builtin_i.trap] = builtin_process.Trap(sig_state, exec_deps.traps,
exec_deps.trap_nodes, ex,
errfmt)
# History evaluation is a no-op if line_input is None.
hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)
if opts.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
mutable_opts.set_interactive()
elif opts.i: # force interactive
arena.PushSource(source.Stdin(' -i'))
line_reader = py_reader.InteractiveLineReader(arena, prompt_ev,
hist_ev, line_input,
prompt_state)
mutable_opts.set_interactive()
else:
script_name = arg_r.Peek()
if script_name is None:
if sys.stdin.isatty():
arena.PushSource(source.Interactive())
line_reader = py_reader.InteractiveLineReader(
arena, prompt_ev, hist_ev, line_input, prompt_state)
mutable_opts.set_interactive()
else:
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
except OSError as e:
ui.Stderr("osh: Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
c_parser = parse_ctx.MakeOshParser(line_reader)
if exec_opts.interactive():
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = passwd.GetMyHomeDir()
assert home_dir is not None
rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil',
lang + 'rc')
history_filename = os_path.join(home_dir, '.config/oil',
'history_' + lang)
if line_input:
# NOTE: We're using a different WordEvaluator here.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
errfmt)
ev.arith_ev = arith_ev
ev.expr_ev = expr_ev
ev.prompt_ev = prompt_ev
ev.CheckCircularDeps()
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
compopt_state, comp_ui_state,
comp_ctx, debug_f)
term_width = 0
if opts.completion_display == 'nice':
try:
term_width = libc.get_terminal_width()
except IOError: # stdin not a terminal
pass
if term_width != 0:
display = comp_ui.NiceDisplay(term_width, comp_ui_state,
prompt_state, debug_f,
line_input)
else:
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
_InitReadline(line_input, history_filename, root_comp, display,
debug_f)
_InitDefaultCompletions(ex, complete_builtin, comp_lookup)
else: # Without readline module
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
sig_state.InitInteractiveShell(display)
# NOTE: Call this AFTER _InitDefaultCompletions.
try:
SourceStartupFile(rc_path, lang, parse_ctx, ex)
except util.UserExit as e:
return e.status
line_reader.Reset() # After sourcing startup file, render $PS1
prompt_plugin = prompt.UserPlugin(mem, parse_ctx, ex)
try:
status = main_loop.Interactive(opts, ex, c_parser, display,
prompt_plugin, errfmt)
if ex.MaybeRunExitTrap():
status = ex.LastStatus()
except util.UserExit as e:
status = e.status
return status
nodes_out = [] if exec_opts.noexec() else None
if nodes_out is None and opts.parser_mem_dump:
raise args.UsageError('--parser-mem-dump can only be used with -n')
_tlog('Execute(node)')
try:
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
if ex.MaybeRunExitTrap():
status = ex.LastStatus()
except util.UserExit as e:
status = e.status
# Only print nodes if the whole parse succeeded.
if nodes_out is not None and status == 0:
if opts.parser_mem_dump: # only valid in -n mode
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def Main(lang, arg_r, environ, login_shell, loader, line_input):
# type: (str, args.Reader, Dict[str, str], bool, pyutil._ResourceLoader, Any) -> int
"""The full shell lifecycle. Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, arg_r: command line arguments
environ: environment
login_shell: Was - on the front?
loader: to get help, version, grammar, etc.
line_input: optional GNU readline
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - shopt -s oil:all
# - Change the prompt in the interactive shell?
# osh-pure:
# - no oil grammar
# - no expression evaluator
# - no interactive shell, or line_input
# - no process.*
# process.{ExternalProgram,Waiter,FdState,JobState,SignalState} -- we want
# to evaluate config files without any of these
# Modules not translated yet: completion, comp_ui, builtin_comp, process
# - word evaluator
# - shouldn't glob? set -o noglob? or hard failure?
# - ~ shouldn't read from the file system
# - I guess it can just be the HOME=HOME?
# Builtin:
# shellvm -c 'echo hi'
# shellvm <<< 'echo hi'
argv0 = arg_r.Peek()
assert argv0 is not None
arg_r.Next()
assert lang in ('osh', 'oil'), lang
try:
attrs = flag_spec.ParseMore('main', arg_r)
except error.Usage as e:
stderr_line('osh usage error: %s', e.msg)
return 2
flag = arg_types.main(attrs.attrs)
arena = alloc.Arena()
errfmt = ui.ErrorFormatter(arena)
help_builtin = builtin_misc.Help(loader, errfmt)
if flag.help:
help_builtin.Run(MakeBuiltinArgv(['%s-usage' % lang]))
return 0
if flag.version:
# OSH version is the only binary in Oil right now, so it's all one version.
pyutil.ShowAppVersion('Oil', loader)
return 0
no_str = None # type: str
debug_stack = [] # type: List[state.DebugFrame]
if arg_r.AtEnd():
dollar0 = argv0
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
# Copy quirky bash behavior.
frame0 = state.DebugFrame(dollar0, 'main', no_str, state.LINE_ZERO, 0,
0)
debug_stack.append(frame0)
# Copy quirky bash behavior.
frame1 = state.DebugFrame(no_str, no_str, no_str, runtime.NO_SPID, 0, 0)
debug_stack.append(frame1)
script_name = arg_r.Peek() # type: Optional[str]
arg_r.Next()
mem = state.Mem(dollar0, arg_r.Rest(), arena, debug_stack)
version_str = pyutil.GetVersion(loader)
state.InitMem(mem, environ, version_str)
procs = {} # type: Dict[str, command__ShFunction]
job_state = process.JobState()
fd_state = process.FdState(errfmt, job_state, mem)
opt_hook = state.OptHook()
parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
# TODO: only MutableOpts needs mem, so it's not a true circular dep.
mem.exec_opts = exec_opts # circular dep
if attrs.show_options: # special case: sh -o
mutable_opts.ShowOptions([])
return 0
# Set these BEFORE processing flags, so they can be overridden.
if lang == 'oil':
mutable_opts.SetShoptOption('oil:all', True)
builtin_pure.SetShellOpts(mutable_opts, attrs.opt_changes,
attrs.shopt_changes)
# feedback between runtime and parser
aliases = {} # type: Dict[str, str]
oil_grammar = None # type: grammar.Grammar
#oil_grammar = pyutil.LoadOilGrammar(loader)
if flag.one_pass_parse and not exec_opts.noexec():
e_usage('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
parse_ctx.Init_OnePassParse(flag.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena, parse_opts, aliases,
oil_grammar)
comp_ctx.Init_Trail(trail1)
comp_ctx.Init_OnePassParse(True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
oil_grammar)
hist_ctx.Init_Trail(trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - cmd_ev and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - cmd_ev and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
cmd_deps = cmd_eval.Deps()
cmd_deps.mutable_opts = mutable_opts
# TODO: In general, cmd_deps are shared between the mutually recursive
# evaluators. Some of the four below are only shared between a builtin and
# the CommandEvaluator, so we could put them somewhere else.
cmd_deps.traps = {}
cmd_deps.trap_nodes = [] # TODO: Clear on fork() to avoid duplicates
waiter = process.Waiter(job_state, exec_opts)
my_pid = posix.getpid()
debug_path = ''
debug_dir = environ.get('OSH_DEBUG_DIR')
if flag.debug_file is not None:
# --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = flag.debug_file
elif debug_dir is not None:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if len(debug_path):
raise NotImplementedError()
else:
debug_f = util.NullDebugFile() # type: util._DebugFile
cmd_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
if mylib.PYTHON:
iso_stamp = time_.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid,
arg_r.argv)
if len(debug_path):
debug_f.log('Writing logs to %r', debug_path)
interp = environ.get('OSH_HIJACK_SHEBANG', '')
search_path = state.SearchPath(mem)
ext_prog = process.ExternalProgram(interp, fd_state, errfmt, debug_f)
splitter = split.SplitContext(mem)
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = environ.get('OSH_CRASH_DUMP_DIR', '')
cmd_deps.dumper = dev.CrashDumper(crash_dump_dir)
if flag.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(mylib.Stderr())
#comp_lookup = completion.Lookup()
# Various Global State objects to work around readline interfaces
#compopt_state = completion.OptionState()
#comp_ui_state = comp_ui.State()
#prompt_state = comp_ui.PromptState()
dir_stack = state.DirStack()
#
# Initialize builtins that don't depend on evaluators
#
builtins = {} # type: Dict[int, vm._Builtin]
AddPure(builtins, mem, procs, mutable_opts, aliases, search_path, errfmt)
AddIO(builtins, mem, dir_stack, exec_opts, splitter, parse_ctx, errfmt)
builtins[builtin_i.help] = help_builtin
#
# Initialize Evaluators
#
arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, parse_ctx, errfmt)
bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, parse_ctx, errfmt)
expr_ev = None # type: expr_eval.OilEvaluator
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)
assign_b = InitAssignmentBuiltins(mem, procs, errfmt)
cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
arena, cmd_deps)
shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
builtins, search_path, ext_prog, waiter,
job_state, fd_state, errfmt)
#shell_ex = NullExecutor(exec_opts, mutable_opts, procs, builtins)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, mem)
tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
trace_f)
# Wire up circular dependencies.
vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
prompt_ev, tracer)
#
# Initialize builtins that depend on evaluators
#
# note: 'printf -v a[i]' and 'unset a[i]' require same deps
builtins[builtin_i.printf] = builtin_printf.Printf(mem, exec_opts,
parse_ctx, arith_ev,
errfmt)
builtins[builtin_i.unset] = builtin_assign.Unset(mem, exec_opts, procs,
parse_ctx, arith_ev,
errfmt)
builtins[builtin_i.eval] = builtin_meta.Eval(parse_ctx, exec_opts, cmd_ev)
#source_builtin = builtin_meta.Source(parse_ctx, search_path, cmd_ev,
#fd_state, errfmt)
#builtins[builtin_i.source] = source_builtin
#builtins[builtin_i.dot] = source_builtin
AddMeta(builtins, shell_ex, mutable_opts, mem, procs, aliases, search_path,
errfmt)
AddBlock(builtins, mem, mutable_opts, dir_stack, cmd_ev, errfmt)
#sig_state = process.SignalState()
#sig_state.InitShell()
#builtins[builtin_i.trap] = builtin_process.Trap(sig_state, cmd_deps.traps,
# cmd_deps.trap_nodes,
# parse_ctx, errfmt)
if flag.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(flag.c,
arena) # type: reader._Reader
if flag.i: # -c and -i can be combined
mutable_opts.set_interactive()
elif flag.i: # force interactive
raise NotImplementedError()
else:
if script_name is None:
stdin = mylib.Stdin()
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
#f = mylib.open(script_name)
except OSError as e:
stderr_line("osh: Couldn't open %r: %s", script_name,
pyutil.strerror(e))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
c_parser = parse_ctx.MakeOshParser(line_reader)
if exec_opts.interactive():
raise NotImplementedError()
if exec_opts.noexec():
status = 0
try:
node = main_loop.ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
status = 2
if status == 0:
if flag.parser_mem_dump is not None: # only valid in -n mode
input_path = '/proc/%d/status' % posix.getpid()
pyutil.CopyFile(input_path, flag.parser_mem_dump)
ui.PrintAst(node, flag)
else:
if flag.parser_mem_dump is not None:
e_usage('--parser-mem-dump can only be used with -n')
try:
status = main_loop.Batch(cmd_ev,
c_parser,
arena,
cmd_flags=cmd_eval.IsMainProgram)
except util.UserExit as e:
status = e.status
box = [status]
cmd_ev.MaybeRunExitTrap(box)
status = box[0]
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if flag.runtime_mem_dump is not None:
input_path = '/proc/%d/status' % posix.getpid()
pyutil.CopyFile(input_path, flag.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
