def ExecExternalProgram(argv, environ):
"""Execute a program and exit this process.
Called by:
ls /
exec ls /
( ls / )
"""
# TODO: If there is an error, like the file isn't executable, then we should
# exit, and the parent will reap it. Should it capture stderr?
try:
os_.execvpe(argv[0], argv, environ)
except OSError as e:
util.error('%r: %s', argv[0], posix.strerror(e.errno))
# POSIX mentions 126 and 127 for two specific errors. The rest are
# unspecified.
#
# http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
if e.errno == errno.EACCES:
status = 126
elif e.errno == errno.ENOENT:
status = 127 # e.g. command not found should be 127.
else:
# dash uses 2, but we use that for parse errors. This seems to be
# consistent with mksh and zsh.
status = 127
sys.exit(status)
def Cd(argv, mem, dir_stack):
arg, i = CD_SPEC.Parse(argv)
# TODO: error checking, etc.
# TODO: ensure that if multiple flags are provided, the *last* one overrides
# the others.
try:
dest_dir = argv[i]
except IndexError:
val = mem.GetVar('HOME')
if val.tag == value_e.Undef:
util.error("$HOME isn't defined")
return 1
elif val.tag == value_e.Str:
dest_dir = val.s
elif val.tag == value_e.StrArray:
util.error("$HOME shouldn't be an array.")
return 1
if dest_dir == '-':
old = mem.GetVar('OLDPWD', scope_e.GlobalOnly)
if old.tag == value_e.Undef:
log('OLDPWD not set')
return 1
elif old.tag == value_e.Str:
dest_dir = old.s
print(dest_dir) # Shells print the directory
elif old.tag == value_e.StrArray:
# TODO: Prevent the user from setting OLDPWD to array (or maybe they
# can't even set it at all.)
raise AssertionError('Invalid OLDPWD')
pwd = mem.GetVar('PWD')
assert pwd.tag == value_e.Str, pwd # TODO: Need a general scheme to avoid
# Calculate new directory, chdir() to it, then set PWD to it. NOTE: We can't
# call posix.getcwd() because it can raise OSError if the directory was
# removed (ENOENT.)
abspath = os_path.join(pwd.s, dest_dir) # make it absolute, for cd ..
if arg.P:
# -P means resolve symbolic links, then process '..'
real_dest_dir = libc.realpath(abspath)
else:
# -L means process '..' first. This just does string manipulation. (But
# realpath afterward isn't correct?)
real_dest_dir = os_path.normpath(abspath)
try:
posix.chdir(real_dest_dir)
except OSError as e:
# TODO: Add line number, etc.
util.error("cd %r: %s", real_dest_dir, posix.strerror(e.errno))
return 1
state.ExportGlobalString(mem, 'OLDPWD', pwd.s)
state.ExportGlobalString(mem, 'PWD', real_dest_dir)
dir_stack.Reset() # for pushd/popd/dirs
return 0
def Exec(self, argv, environ):
"""Execute a program and exit this process.
Called by:
ls /
exec ls /
( ls / )
"""
if self.hijack_shebang:
try:
f = self.fd_state.Open(argv[0])
except OSError as e:
pass
else:
try:
line = f.read(40)
if (line.startswith('#!/bin/sh')
or line.startswith('#!/bin/bash')
or line.startswith('#!/usr/bin/env bash')):
self.debug_f.log('Hijacked: %s with %s', argv,
self.hijack_shebang)
argv = [self.hijack_shebang] + argv
else:
#self.debug_f.log('Not hijacking %s (%r)', argv, line)
pass
finally:
f.close()
# TODO: If there is an error, like the file isn't executable, then we should
# exit, and the parent will reap it. Should it capture stderr?
try:
os_.execvpe(argv[0], argv, environ)
except OSError as e:
# TODO: Run with /bin/sh when ENOEXEC error (noshebang). Because all
# shells do it.
util.error('%r: %s', argv[0], posix.strerror(e.errno))
# POSIX mentions 126 and 127 for two specific errors. The rest are
# unspecified.
#
# http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
if e.errno == errno.EACCES:
status = 126
elif e.errno == errno.ENOENT:
status = 127 # e.g. command not found should be 127.
else:
# dash uses 2, but we use that for parse errors. This seems to be
# consistent with mksh and zsh.
status = 127
sys.exit(status) # raises SystemExit
def Popd(argv, home_dir, dir_stack):
dest_dir = dir_stack.Pop()
if dest_dir is None:
util.error('popd: directory stack is empty')
return 1
try:
posix.chdir(dest_dir)
except OSError as e:
util.error("popd: %r: %s", dest_dir, posix.strerror(e.errno))
return 1
_PrintDirStack(dir_stack, SINGLE_LINE, home_dir)
return 0
def Popd(argv, mem, dir_stack):
dest_dir = dir_stack.Pop()
if dest_dir is None:
util.error('popd: directory stack is empty')
return 1
try:
posix.chdir(dest_dir)
except OSError as e:
util.error("popd: %r: %s", dest_dir, posix.strerror(e.errno))
return 1
_PrintDirStack(dir_stack, SINGLE_LINE, mem.GetVar('HOME'))
state.SetGlobalString(mem, 'PWD', dest_dir)
return 0
def EvalForPlugin(self, w):
"""Wrapper around EvalWordToString that prevents errors.
Runtime errors like $(( 1 / 0 )) and mutating $? like $(exit 42) are
handled here.
"""
self.mem.PushStatusFrame() # to "sandbox" $? and $PIPESTATUS
try:
val = self.EvalWordToString(w)
except util.FatalRuntimeError as e:
val = value.Str("<Runtime error: %s>" % e.UserErrorString())
except (OSError, IOError) as e:
# This is like the catch-all in Executor.ExecuteAndCatch().
val = value.Str("<I/O error: %s>" % posix.strerror(e.errno))
finally:
self.mem.PopStatusFrame()
return val
def Pushd(argv, home_dir, dir_stack):
num_args = len(argv)
if num_args <= 0:
util.error('pushd: no other directory')
return 1
elif num_args > 1:
util.error('pushd: too many arguments')
return 1
dest_dir = os_path.abspath(argv[0])
try:
posix.chdir(dest_dir)
except OSError as e:
util.error("pushd: %r: %s", dest_dir, posix.strerror(e.errno))
return 1
dir_stack.Push(dest_dir)
_PrintDirStack(dir_stack, SINGLE_LINE, home_dir)
return 0
def _PushDup(self, fd1, fd2):
"""Save fd2, and dup fd1 onto fd2.
Mutates self.cur_frame.saved.
Returns:
success Bool
"""
new_fd = self._NextFreeFileDescriptor()
#log('---- _PushDup %s %s', fd1, fd2)
need_restore = True
try:
#log('DUPFD %s %s', fd2, self.next_fd)
fcntl.fcntl(fd2, fcntl.F_DUPFD, new_fd)
except IOError as e:
# Example program that causes this error: exec 4>&1. Descriptor 4 isn't
# open.
# This seems to be ignored in dash too in savefd()?
if e.errno == errno.EBADF:
#log('ERROR %s', e)
need_restore = False
else:
raise
else:
posix.close(fd2)
fcntl.fcntl(new_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
#log('==== dup %s %s\n' % (fd1, fd2))
try:
posix.dup2(fd1, fd2)
except OSError as e:
# bash/dash give this error too, e.g. for 'echo hi 1>&3'
util.error('%d: %s', fd1, posix.strerror(e.errno))
# Restore and return error
posix.dup2(new_fd, fd2)
posix.close(new_fd)
# Undo it
return False
if need_restore:
self.cur_frame.saved.append((new_fd, fd2))
return True
def _Source(self, argv):
try:
path = argv[0]
except IndexError:
# TODO: Should point to the source statement that failed.
util.error('source: missing required argument')
return 1
try:
f = self.fd_state.Open(path) # Shell can't use descriptors 3-9
except OSError as e:
# TODO: Should point to the source statement that failed.
util.error('source %r failed: %s', path, posix.strerror(e.errno))
return 1
try:
line_reader = reader.FileLineReader(f, self.arena)
_, c_parser = self.parse_ctx.MakeParser(line_reader)
# A sourced module CAN have a new arguments array, but it always shares
# the same variable scope as the caller. The caller could be at either a
# global or a local scope.
source_argv = argv[1:]
self.mem.PushSource(path, source_argv)
try:
status = self._EvalHelper(c_parser, path)
finally:
self.mem.PopSource(source_argv)
return status
except _ControlFlow as e:
if e.IsReturn():
return e.StatusCode()
else:
raise
finally:
f.close()
def test_strerror(self):
if hasattr(posix, 'strerror'):
self.assert_(posix.strerror(0))
def OshCommandMain(argv):
"""Run an 'oshc' tool.
'osh' is short for "osh compiler" or "osh command".
TODO:
- oshc --help
oshc deps
--path: the $PATH to use to find executables. What about libraries?
NOTE: we're leaving out su -c, find, xargs, etc.? Those should generally
run functions using the $0 pattern.
--chained-command sudo
"""
try:
action = argv[0]
except IndexError:
raise args.UsageError('oshc: Missing required subcommand.')
if action not in SUBCOMMANDS:
raise args.UsageError('oshc: Invalid subcommand %r.' % action)
try:
script_name = argv[1]
except IndexError:
script_name = '<stdin>'
f = sys.stdin
else:
try:
f = open(script_name)
except IOError as e:
util.error("Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 2
pool = alloc.Pool()
arena = pool.NewArena()
arena.PushSource(script_name)
line_reader = reader.FileLineReader(f, arena)
aliases = {} # Dummy value; not respecting aliases!
parse_ctx = parse_lib.ParseContext(arena, aliases)
c_parser = parse_ctx.MakeOshParser(line_reader)
try:
node = main_loop.ParseWholeFile(c_parser)
except util.ParseError as e:
ui.PrettyPrintError(e, arena, sys.stderr)
return 2
assert node is not None
f.close()
# Columns for list-*
# path line name
# where name is the binary path, variable name, or library path.
# bin-deps and lib-deps can be used to make an app bundle.
# Maybe I should list them together? 'deps' can show 4 columns?
#
# path, line, type, name
#
# --pretty can show the LST location.
# stderr: show how we're following imports?
if action == 'translate':
osh2oil.PrintAsOil(arena, node)
elif action == 'arena': # for debugging
osh2oil.PrintArena(arena)
elif action == 'spans': # for debugging
osh2oil.PrintSpans(arena)
elif action == 'format':
# TODO: autoformat code
raise NotImplementedError(action)
elif action == 'deps':
deps.Deps(node)
elif action == 'undefined-vars': # could be environment variables
raise NotImplementedError
else:
raise AssertionError # Checked above
return 0
def ShellMain(lang, argv0, argv, login_shell):
"""Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox.
login_shell: Was - on the front?
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - the parser and executor
# - Change the prompt in the interactive shell?
assert lang in ('osh', 'oil'), lang
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.usage('osh usage error: %s', e)
return 2
if opts.help:
loader = util.GetResourceLoader()
builtin.Help(['%s-usage' % lang], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
# TODO: This should be in interactive mode only?
builtin.RegisterSigIntHandler()
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
pool = alloc.Pool()
arena = pool.NewArena()
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
funcs = {}
fd_state = process.FdState()
exec_opts = state.ExecOpts(mem, readline)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
aliases = {} # feedback between runtime and parser
parse_ctx = parse_lib.ParseContext(arena, aliases) # For main_loop
# Three ParseContext instances SHARE aliases. TODO: Complete aliases.
comp_arena = pool.NewArena()
comp_arena.PushSource('<completion>')
trail1 = parse_lib.Trail()
comp_ctx = parse_lib.ParseContext(comp_arena, aliases, trail=trail1)
hist_arena = pool.NewArena()
hist_arena.PushSource('<history>')
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - ex and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - ex and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
exec_deps = cmd_exec.Deps()
if opts.debug_file:
debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w'))
else:
debug_f = util.NullDebugFile()
exec_deps.debug_f = debug_f
debug_f.log('Debug file is %s', opts.debug_file)
splitter = split.SplitContext(mem)
exec_deps.splitter = splitter
# Controlled by env variable, flag, or hook?
exec_deps.dumper = dev.CrashDumper(
posix.environ.get('OSH_CRASH_DUMP_DIR', ''))
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
exec_deps.trace_f = trace_f
# TODO: Separate comp_state and comp_lookup.
comp_state = completion.State()
comp_lookup = completion.Lookup()
builtins = { # Lookup
builtin_e.HISTORY: builtin.History(readline),
builtin_e.COMPOPT: builtin_comp.CompOpt(comp_state),
builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
}
ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
parse_ctx, exec_deps)
exec_deps.ex = ex
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
exec_deps.word_ev = word_ev
arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.arith_ev = arith_ev
word_ev.arith_ev = arith_ev # Another circular dependency
bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.bool_ev = bool_ev
tracer = cmd_exec.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
exec_deps.tracer = tracer
# HACK for circular deps
ex.word_ev = word_ev
ex.arith_ev = arith_ev
ex.bool_ev = bool_ev
ex.tracer = tracer
spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter)
# Add some builtins that depend on the executor!
complete_builtin = builtin_comp.Complete(spec_builder,
comp_lookup) # used later
builtins[builtin_e.COMPLETE] = complete_builtin
builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)
if lang == 'oil':
# The Oil executor wraps an OSH executor? It needs to be able to source
# it.
ex = oil_cmd_exec.OilExecutor(ex)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = ui.PromptEvaluator(lang, arena, parse_ctx, ex, mem)
exec_deps.prompt_ev = prompt_ev
word_ev.prompt_ev = prompt_ev # HACK for circular deps
# History evaluation is a no-op if readline is None.
hist_ev = reader.HistoryEvaluator(readline, hist_ctx, debug_f)
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = mem.GetVar('HOME')
assert home_dir.tag == value_e.Str, home_dir
rc_path = opts.rcfile or os_path.join(home_dir.s, '.config/oil',
lang + 'rc')
history_filename = os_path.join(home_dir.s, '.config/oil',
'history_' + lang)
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
# interactive shell only
line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
# interactive shell only
line_reader = reader.InteractiveLineReader(
arena, prompt_ev, hist_ev)
exec_opts.interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(script_name)
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
if lang == 'osh':
c_parser = parse_ctx.MakeOshParser(line_reader)
else:
c_parser = parse_ctx.MakeOilParser(line_reader)
if exec_opts.interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if readline:
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
arena)
progress_f = ui.StatusLine()
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
comp_state, comp_ctx,
progress_f, debug_f)
_InitReadline(readline, history_filename, root_comp, debug_f)
_InitDefaultCompletions(ex, complete_builtin, comp_lookup)
# NOTE: Call this AFTER _InitDefaultCompletions.
SourceStartupFile(rc_path, lang, parse_ctx, ex)
return main_loop.Interactive(opts, ex, c_parser, arena)
# TODO: Remove this after removing it from benchmarks/osh-runtime. It's no
# longer relevant with main_loop.
if opts.parser_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
nodes_out = [] if exec_opts.noexec else None
_tlog('Execute(node)')
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
# Only print nodes if the whole parse succeeded.
if nodes_out is not None and status == 0:
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: This doesn't cause any spec tests to fail, but it could.
if posix.environ.get('ASDL_TYPE_CHECK'):
log('NOTE: Performed %d ASDL_TYPE_CHECKs.', runtime.NUM_TYPE_CHECKS)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def ShellMain(lang, argv0, argv, login_shell):
"""Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox.
login_shell: Was - on the front?
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - the parser and executor
# - Change the prompt in the interactive shell?
assert lang in ('osh', 'oil'), lang
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.usage('osh usage error: %s', e)
return 2
# NOTE: This has a side effect of deleting _OVM_* from the environment!
# TODO: Thread this throughout the program, and get rid of the global
# variable in core/util.py. Rename to InitResourceLaoder(). It's now only
# used for the 'help' builtin and --version.
loader = pyutil.GetResourceLoader()
if opts.help:
builtin.Help(['%s-usage' % lang], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
arena = alloc.Arena()
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
funcs = {}
fd_state = process.FdState()
exec_opts = state.ExecOpts(mem, line_input)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
aliases = {} # feedback between runtime and parser
if opts.one_pass_parse and not exec_opts.noexec:
raise args.UsageError('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena,
aliases,
one_pass_parse=opts.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena,
aliases,
trail=trail1,
one_pass_parse=True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - ex and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - ex and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
exec_deps = cmd_exec.Deps()
my_pid = posix.getpid()
debug_path = ''
debug_dir = posix.environ.get('OSH_DEBUG_DIR')
if opts.debug_file: # --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = opts.debug_file
elif debug_dir:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if debug_path:
# This will be created as an empty file if it doesn't exist, or it could be
# a pipe.
try:
debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
except OSError as e:
util.error("Couldn't open %r: %s", debug_path,
posix.strerror(e.errno))
return 2
else:
debug_f = util.NullDebugFile()
exec_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
if debug_path:
debug_f.log('Writing logs to %r', debug_path)
interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
exec_deps.ext_prog = process.ExternalProgram(interp, fd_state, debug_f)
splitter = split.SplitContext(mem)
exec_deps.splitter = splitter
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
exec_deps.dumper = dev.CrashDumper(crash_dump_dir)
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
exec_deps.trace_f = trace_f
comp_lookup = completion.Lookup()
#
# Various Global State objects to work around readline interfaces
#
compopt_state = completion.OptionState()
comp_ui_state = comp_ui.State()
prompt_state = comp_ui.PromptState()
builtins = { # Lookup
builtin_e.HISTORY: builtin.History(line_input),
builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state),
builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
}
ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
parse_ctx, exec_deps)
exec_deps.ex = ex
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
exec_deps.word_ev = word_ev
arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.arith_ev = arith_ev
word_ev.arith_ev = arith_ev # Another circular dependency
bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.bool_ev = bool_ev
tracer = dev.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
exec_deps.tracer = tracer
# HACK for circular deps
ex.word_ev = word_ev
ex.arith_ev = arith_ev
ex.bool_ev = bool_ev
ex.tracer = tracer
spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
comp_lookup)
# Add some builtins that depend on the executor!
complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
builtins[builtin_e.COMPLETE] = complete_builtin
builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)
if lang == 'oil':
# The Oil executor wraps an OSH executor? It needs to be able to source
# it.
ex = oil_cmd_exec.OilExecutor(ex)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, ex, mem)
exec_deps.prompt_ev = prompt_ev
word_ev.prompt_ev = prompt_ev # HACK for circular deps
# History evaluation is a no-op if line_input is None.
hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = process.GetHomeDir()
assert home_dir is not None
rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil', lang + 'rc')
history_filename = os_path.join(home_dir, '.config/oil', 'history_' + lang)
if opts.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource(source.Stdin(' -i'))
# interactive shell only
line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
line_input, prompt_state)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource(source.Interactive())
# interactive shell only
line_reader = reader.InteractiveLineReader(
arena, prompt_ev, hist_ev, line_input, prompt_state)
exec_opts.interactive = True
else:
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
if lang == 'osh':
c_parser = parse_ctx.MakeOshParser(line_reader)
else:
c_parser = parse_ctx.MakeOilParser(line_reader)
# NOTE: SIGINT is temporarily enabled during readline() by
# frontend/reader.py.
# It's treated differently than SIGQUIT and SIGTSTP because Python handles it
# with KeyboardInterrupt. We don't want KeyboardInterrupt at arbitrary
# points in a non-interactive shell. (e.g. osh -c 'sleep 5' then Ctrl-C)
signal.signal(signal.SIGINT, signal.SIG_IGN)
if exec_opts.interactive:
if line_input:
# NOTE: We're using a different WordEvaluator here.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
arena)
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
compopt_state, comp_ui_state,
comp_ctx, debug_f)
term_width = 0
if opts.completion_display == 'nice':
try:
term_width = libc.get_terminal_width()
except IOError: # stdin not a terminal
pass
if term_width != 0:
display = comp_ui.NiceDisplay(term_width, comp_ui_state,
prompt_state, debug_f,
line_input)
else:
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
_InitReadline(line_input, history_filename, root_comp, display,
debug_f)
_InitDefaultCompletions(ex, complete_builtin, comp_lookup)
else: # Without readline module
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
# The shell itself should ignore Ctrl-\.
signal.signal(signal.SIGQUIT, signal.SIG_IGN)
# This prevents Ctrl-Z from suspending OSH in interactive mode. But we're
# not getting notification via wait() that the child stopped?
signal.signal(signal.SIGTSTP, signal.SIG_IGN)
# Register a callback to receive terminal width changes.
signal.signal(signal.SIGWINCH, lambda x, y: display.OnWindowChange())
# NOTE: Call this AFTER _InitDefaultCompletions.
SourceStartupFile(rc_path, lang, parse_ctx, ex)
line_reader.Reset() # After sourcing startup file, render $PS1
return main_loop.Interactive(opts, ex, c_parser, display, arena)
nodes_out = [] if exec_opts.noexec else None
if nodes_out is None and opts.parser_mem_dump:
raise args.UsageError('--parser-mem-dump can only be used with -n')
_tlog('Execute(node)')
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
# Only print nodes if the whole parse succeeded.
if nodes_out is not None and status == 0:
if opts.parser_mem_dump: # only valid in -n mode
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
import posix
try:
print(int('abc'))
except ValueError:
print('can\'t convert \'abc\' to an int')
try:
# this fd should never be open in this test
posix.close(0x1000000)
print('somehow managed to close ridiculous fd')
except OSError as e:
print('can\'t close ridiculous fd; errno=' + repr(e.errno) + ' (' +
posix.strerror(e.errno) + ')')
def write(self, s):
raise IOError(errno.EBADF, posix.strerror(errno.EBADF))
def OshMain(argv0, argv, login_shell):
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.usage('osh usage error: %s', e)
return 2
if opts.help:
loader = util.GetResourceLoader()
builtin.Help(['osh-usage'], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
# TODO: This should be in interactive mode only?
builtin.RegisterSigIntHandler()
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
pool = alloc.Pool()
arena = pool.NewArena()
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
funcs = {}
comp_lookup = completion.CompletionLookup()
fd_state = process.FdState()
exec_opts = state.ExecOpts(mem, readline)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
aliases = {} # feedback between runtime and parser
parse_ctx = parse_lib.ParseContext(arena, aliases)
if opts.debug_file:
debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w'))
else:
debug_f = util.NullDebugFile()
debug_f.log('Debug file is %s', opts.debug_file)
# Controlled by env variable, flag, or hook?
dumper = dev.CrashDumper(posix.environ.get('OSH_CRASH_DUMP_DIR', ''))
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
devtools = dev.DevTools(dumper, debug_f, trace_f)
ex = cmd_exec.Executor(mem, fd_state, funcs, comp_lookup, exec_opts,
parse_ctx, devtools)
# NOTE: The rc file can contain both commands and functions... ideally we
# would only want to save nodes/lines for the functions.
try:
rc_path = 'oilrc'
arena.PushSource(rc_path)
with open(rc_path) as f:
rc_line_reader = reader.FileLineReader(f, arena)
_, rc_c_parser = parse_ctx.MakeParser(rc_line_reader)
try:
status = main_loop.Batch(ex, rc_c_parser, arena)
finally:
arena.PopSource()
except IOError as e:
if e.errno != errno.ENOENT:
raise
# Needed in non-interactive shells for @P
prompt = ui.Prompt(arena, parse_ctx, ex)
ui.PROMPT = prompt
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
line_reader = reader.InteractiveLineReader(arena, prompt)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
line_reader = reader.InteractiveLineReader(arena, prompt)
exec_opts.interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(script_name)
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
w_parser, c_parser = parse_ctx.MakeParser(line_reader)
if exec_opts.interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if HAVE_READLINE:
splitter = legacy.SplitContext(mem) # TODO: share with executor.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
arena)
progress_f = ui.StatusLine()
var_action = completion.VariablesActionInternal(ex.mem)
root_comp = completion.RootCompleter(ev, comp_lookup, var_action,
parse_ctx, progress_f,
debug_f)
completion.Init(readline, root_comp, debug_f)
_InitDefaultCompletions(ex, comp_lookup)
return main_loop.Interactive(opts, ex, c_parser, arena)
# TODO: Remove this after removing it from benchmarks/osh-runtime. It's no
# longer relevant with main_loop.
if opts.parser_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
nodes_out = [] if exec_opts.noexec else None
_tlog('Execute(node)')
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
if nodes_out is not None:
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def _ApplyRedirect(self, r, waiter):
ok = True
if r.tag == redirect_e.PathRedirect:
if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
# NOTE: This is different than >| because it respects noclobber, but
# that option is almost never used. See test/wild.sh.
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id == Id.Redir_Clobber: # >|
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_TRUNC
elif r.op_id in (Id.Redir_DGreat, Id.Redir_AndDGreat): # >> &>>
mode = posix.O_CREAT | posix.O_WRONLY | posix.O_APPEND
elif r.op_id == Id.Redir_Less: # <
mode = posix.O_RDONLY
else:
raise NotImplementedError(r.op_id)
# NOTE: 0666 is affected by umask, all shells use it.
try:
target_fd = posix.open(r.filename, mode, 0666)
except OSError as e:
util.error("Can't open %r: %s", r.filename,
posix.strerror(e.errno))
return False
# Apply redirect
if not self._PushDup(target_fd, r.fd):
ok = False
# Now handle the extra redirects for aliases &> and &>>.
#
# We can rewrite
# stdout_stderr.py &> out-err.txt
# as
# stdout_stderr.py > out-err.txt 2>&1
#
# And rewrite
# stdout_stderr.py 3&> out-err.txt
# as
# stdout_stderr.py 3> out-err.txt 2>&3
if ok:
if r.op_id == Id.Redir_AndGreat:
if not self._PushDup(r.fd, 2):
ok = False
elif r.op_id == Id.Redir_AndDGreat:
if not self._PushDup(r.fd, 2):
ok = False
posix.close(target_fd) # We already made a copy of it.
# I don't think we need to close(0) because it will be restored from its
# saved position (10), which closes it.
#self._PushClose(r.fd)
elif r.tag == redirect_e.DescRedirect: # e.g. echo hi 1>&2
if r.op_id == Id.Redir_GreatAnd: # 1>&2
if not self._PushDup(r.target_fd, r.fd):
ok = False
elif r.op_id == Id.Redir_LessAnd: # 0<&5
# The only difference between >& and <& is the default file
# descriptor argument.
if not self._PushDup(r.target_fd, r.fd):
ok = False
else:
raise NotImplementedError
elif r.tag == redirect_e.HereRedirect:
# NOTE: Do these descriptors have to be moved out of the range 0-9?
read_fd, write_fd = posix.pipe()
if not self._PushDup(read_fd, r.fd): # stdin is now the pipe
ok = False
# We can't close like we do in the filename case above? The writer can
# get a "broken pipe".
self._PushClose(read_fd)
thunk = _HereDocWriterThunk(write_fd, r.body)
# TODO: Use PIPE_SIZE to save a process in the case of small here docs,
# which are the common case. (dash does this.)
start_process = True
#start_process = False
if start_process:
here_proc = Process(thunk)
# NOTE: we could close the read pipe here, but it doesn't really
# matter because we control the code.
# here_proc.StateChange()
pid = here_proc.Start()
# no-op callback
waiter.Register(pid, here_proc.WhenDone)
#log('Started %s as %d', here_proc, pid)
self._PushWait(here_proc, waiter)
# Now that we've started the child, close it in the parent.
posix.close(write_fd)
else:
posix.write(write_fd, r.body)
posix.close(write_fd)
return ok
