def testStringLineReader(self):
arena = test_lib.MakeArena('<reader_test.py>')
r = reader.StringLineReader('one\ntwo', arena)
self.assertEqual((0, 'one\n', 0), r.GetLine())
self.assertEqual((1, 'two', 0), r.GetLine())
self.assertEqual((-1, None, 0), r.GetLine())
def Run(self, cmd_val):
# type: (cmd_value__Argv) -> int
# There are no flags, but we need it to respect --
_, arg_r = flag_spec.ParseCmdVal('eval', cmd_val)
if self.exec_opts.simple_eval_builtin():
code_str, eval_spid = arg_r.ReadRequired2('requires code string')
if not arg_r.AtEnd():
e_usage('requires exactly 1 argument')
else:
code_str = ' '.join(arg_r.Rest())
# code_str could be EMPTY, so just use the first one
eval_spid = cmd_val.arg_spids[0]
line_reader = reader.StringLineReader(code_str, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
src = source.EvalArg(eval_spid)
with dev.ctx_Tracer(self.tracer, 'eval', None):
with alloc.ctx_Location(self.arena, src):
return main_loop.Batch(self.cmd_ev,
c_parser,
self.arena,
cmd_flags=cmd_eval.IsEvalSource)
def Run(self, cmd_val):
# type: (cmd_value__Argv) -> int
# There are no flags, but we need it to respect --
arg_r = args.Reader(cmd_val.argv, spids=cmd_val.arg_spids)
arg_r.Next() # skip 'eval'
arg = EVAL_SPEC.Parse(arg_r)
if self.exec_opts.strict_eval_builtin():
code_str, eval_spid = arg_r.ReadRequired2('requires code string')
if not arg_r.AtEnd():
raise error.Usage('requires exactly 1 argument')
else:
code_str = ' '.join(cmd_val.argv[arg_r.i:])
# code_str could be EMPTY, so just use the first one
eval_spid = cmd_val.arg_spids[0]
line_reader = reader.StringLineReader(code_str, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
src = source.EvalArg(eval_spid)
self.arena.PushSource(src)
try:
return main_loop.Batch(self.cmd_ev, c_parser, self.arena)
finally:
self.arena.PopSource()
def Run(self):
# type: () -> None
val = self.mem.GetValue('PROMPT_COMMAND')
if val.tag_() != value_e.Str:
return
# PROMPT_COMMAND almost never changes, so we try to cache its parsing.
# This avoids memory allocations.
prompt_cmd = cast(value__Str, val).s
node = self.parse_cache.get(prompt_cmd)
if node is None:
line_reader = reader.StringLineReader(prompt_cmd, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
# NOTE: This is similar to CommandEvaluator.ParseTrapCode().
# TODO: Add spid
with alloc.ctx_Location(self.arena,
source.PromptCommand(runtime.NO_SPID)):
try:
node = main_loop.ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, self.arena)
return # don't execute
self.parse_cache[prompt_cmd] = node
# Save this so PROMPT_COMMAND can't set $?
with state.ctx_Status(self.mem):
# Catches fatal execution error
self.cmd_ev.ExecuteAndCatch(node)
def MakeWordParserForPlugin(self, code_str, arena):
"""FOr $PS1, etc.
NOTE: Uses its own arena! I think that does nothing though?
"""
line_reader = reader.StringLineReader(code_str, arena)
lx = self._MakeLexer(line_reader, arena=arena)
return word_parse.WordParser(self, lx, line_reader)
def MakeArithParser(self, code_str): # type: (str) -> TdopParser """Used for a[x+1]=foo in the CommandParser.""" line_reader = reader.StringLineReader(code_str, self.arena) lx = self._MakeLexer(line_reader) w_parser = word_parse.WordParser(self, lx, line_reader) w_parser.Init(lex_mode_e.Arith) # Special initialization a_parser = tdop.TdopParser(arith_parse.Spec(), w_parser, self.parse_opts) return a_parser
def MakeArithParser(self, code_str):
# type: (str) -> TdopParser
"""Used for a[x+1]=foo in the CommandParser."""
line_reader = reader.StringLineReader(code_str, self.arena)
lx = self._MakeLexer(line_reader)
w_parser = word_parse.WordParser(self,
lx,
line_reader,
lex_mode=lex_mode_e.Arith)
a_parser = tdop.TdopParser(arith_parse.SPEC, w_parser)
return a_parser
def _ParseOsh(self, code_str):
"""Parse a line of OSH, which can include Oil assignments."""
line_reader = reader.StringLineReader(code_str, self.arena)
# the OSH parser hooks into the Oil parser
c_parser = self.parse_ctx.MakeOshParser(line_reader)
node = c_parser.ParseLogicalLine()
print('')
log('\t%s', code_str)
node.PrettyPrint()
print('')
return node
def _Eval(self, arg_vec):
# TODO:
# - set -o sane-eval should change eval to take a single string.
code_str = ' '.join(arg_vec.strs[1:])
eval_spid = arg_vec.spids[0]
line_reader = reader.StringLineReader(code_str, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
src = source.EvalArg(eval_spid)
return self._EvalHelper(c_parser, src)
def MakeArithParser(self, code_str, arena):
"""Used for a[x+1]=foo in the CommandParser.
NOTE: We add tokens to a different arena, so we don't mess up the
invariants for translation.
"""
line_reader = reader.StringLineReader(code_str, arena)
lx = self._MakeLexer(line_reader, arena=arena)
w_parser = word_parse.WordParser(self, lx, line_reader,
lex_mode=lex_mode_e.Arith)
a_parser = tdop.TdopParser(arith_parse.SPEC, w_parser)
return a_parser
def _Eval(self, argv, eval_spid):
# TODO:
# - (argv, spid) should be a pattern for all builtins? They all will need
# to report usage errors.
# - set -o sane-eval should change eval to take a single string.
code_str = ' '.join(argv)
line_reader = reader.StringLineReader(code_str, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
span = self.arena.GetLineSpan(eval_spid)
path, line_num = self.arena.GetDebugInfo(span.line_id)
source_name = '<eval string from %s:%d>' % (path, line_num)
return self._EvalHelper(c_parser, source_name)
def _ParseTrapCode(self, code_str):
# type: (str) -> command_t
"""
Returns:
A node, or None if the code is invalid.
"""
line_reader = reader.StringLineReader(code_str, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
# TODO: the SPID should be passed through argv. Use ArgvWord?
with alloc.ctx_Location(self.arena, source.Trap(runtime.NO_SPID)):
try:
node = main_loop.ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, self.arena)
return None
return node
def ParseDemo(oil_grammar):
# type: (Grammar) -> None
arena = alloc.Arena()
arena.PushSource(source__Stdin(''))
parse_ctx = None # type: ParseContext
e_parser = expr_parse.ExprParser(parse_ctx, oil_grammar)
line_lexer = lexer.LineLexer('', arena)
line_reader = reader.StringLineReader('1 + 2*3', arena)
lex = lexer.Lexer(line_lexer, line_reader)
try:
pnode, _ = e_parser.Parse(lex, arith_nt.arith_expr)
except error.Parse as e:
#ui.PrettyPrintError(e, arena)
log("Parse Error (TODO: print it)")
return
# TODO: Fill this in. Oil uses parse_lib.MakeGrammarNames()
#
# terminals: _Id_str? Doesn't work in mycpp
# nonterminals: gr.number2symbol. Is this ever used at runtime?
#
# Dict[int,str] should really be a List[str] then?
if 0:
names = {} # type: Dict[int, str]
printer = expr_parse.ParseTreePrinter(names)
printer.Print(pnode)
# NOTE: Could also transform
# This only works for Oil
if 0:
tr = expr_to_ast.Transformer(oil_grammar)
node = tr.Expr(pnode)
assert node is not None
tree = node.AbbreviatedTree()
fmt.PrintTree(tree, mylib.Stdout())
def testLineReadersAreEquivalent(self):
a1 = alloc.Arena()
r1 = reader.StringLineReader('one\ntwo', a1)
a2 = alloc.Arena()
f = cStringIO.StringIO('one\ntwo')
r2 = reader.FileLineReader(f, a2)
a3 = alloc.Arena()
lines = [(0, 'one\n', 0), (1, 'two', 0)]
r3 = reader.VirtualLineReader(lines, a3)
for a in [a1, a2, a3]:
a.PushSource(source.MainFile('reader_test.py'))
for r in [r1, r2, r3]:
print(r)
# Lines are added to the arena with a line_id.
self.assertEqual((0, 'one\n', 0), r.GetLine())
self.assertEqual((1, 'two', 0), r.GetLine())
self.assertEqual((-1, None, 0), r.GetLine())
def ParseTrapCode(self, code_str):
"""
Returns:
A node, or None if the code is invalid.
"""
line_reader = reader.StringLineReader(code_str, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
# TODO: the SPID should be passed through argv
self.arena.PushSource(source.Trap(const.NO_INTEGER))
try:
try:
node = main_loop.ParseWholeFile(c_parser)
except util.ParseError as e:
ui.PrettyPrintError(e, self.arena)
return None
finally:
self.arena.PopSource()
return node
def _Line(self, arg, var_name):
# type: (arg_types.read, str) -> int
line = _ReadLine()
if len(line) == 0: # EOF
return 1
if not arg.with_eol:
if line.endswith('\r\n'):
line = line[:-2]
elif line.endswith('\n'):
line = line[:-1]
# Lines that don't start with a single quote aren't QSN. They may contain
# a single quote internally, like:
#
# Fool's Gold
if arg.q and line.startswith("'"):
arena = self.parse_ctx.arena
line_reader = reader.StringLineReader(line, arena)
lexer = self.parse_ctx._MakeLexer(line_reader)
# The parser only yields valid tokens:
# Char_Literals, Char_OneChar, Char_Hex, Char_UBraced
# So we can use word_compile.EvalCStringToken, which is also used for
# $''.
# Important: we don't generate Id.Unknown_Backslash because that is valid
# in echo -e. We just make it Id.Unknown_Tok?
try:
# TODO: read should know about stdin, and redirects, and pipelines?
with alloc.ctx_Location(arena, source.Stdin('')):
tokens = qsn_native.Parse(lexer)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
return 1
tmp = [word_compile.EvalCStringToken(t) for t in tokens]
line = ''.join(tmp)
lhs = lvalue.Named(var_name)
self.mem.SetValue(lhs, value.Str(line), scope_e.LocalOnly)
return 0
def ParseTrapCode(self, code_str):
"""
Returns:
A node, or None if the code is invalid.
"""
line_reader = reader.StringLineReader(code_str, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
source_name = '<trap string>'
self.arena.PushSource(source_name)
try:
try:
node = main_loop.ParseWholeFile(c_parser)
except util.ParseError as e:
util.error('Parse error in %r:', source_name)
ui.PrettyPrintError(e, self.arena)
return None
finally:
self.arena.PopSource()
return node
def ParseDemo(oil_grammar):
# type: (Grammar) -> None
arena = alloc.Arena()
arena.PushSource(source__Stdin(''))
parse_ctx = None # type: ParseContext
e_parser = expr_parse.ExprParser(parse_ctx, oil_grammar)
line_lexer = lexer.LineLexer('', arena)
line_reader = reader.StringLineReader('1 + 2*3', arena)
lex = lexer.Lexer(line_lexer, line_reader)
try:
pnode, _ = e_parser.Parse(lex, arith_nt.arith_expr)
except util.ParseError as e:
#ui.PrettyPrintError(e, arena)
print(e)
return
# TODO: Fill this in. Oil uses parse_lib.MakeGrammarNames()
names = {} # type: Dict[int, str]
printer = expr_parse.ParseTreePrinter(names)
printer.Print(pnode)
def Run(self):
# type: () -> None
val = self.mem.GetVar('PROMPT_COMMAND')
if val.tag != value_e.Str:
return
# PROMPT_COMMAND almost never changes, so we try to cache its parsing.
# This avoids memory allocations.
prompt_cmd = val.s
try:
node = self.parse_cache[prompt_cmd]
except KeyError:
line_reader = reader.StringLineReader(prompt_cmd, self.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader)
# NOTE: This is similar to Executor.ParseTrapCode().
# TODO: Add spid
self.arena.PushSource(source.PromptCommand(const.NO_INTEGER))
try:
try:
node = main_loop.ParseWholeFile(c_parser)
except util.ParseError as e:
ui.PrettyPrintError(e, self.arena)
return # don't execute
finally:
self.arena.PopSource()
self.parse_cache[prompt_cmd] = node
# Save this so PROMPT_COMMAND can't set $?
self.mem.PushStatusFrame()
try:
# Catches fatal execution error
self.ex.ExecuteAndCatch(node)
finally:
self.mem.PopStatusFrame()
def main(argv):
# type: (List[str]) -> int
arena = alloc.Arena()
opt_array = [False] * option_i.ARRAY_SIZE
parse_opts = optview.Parse(opt_array)
# Dummy value; not respecting aliases!
aliases = {} # type: Dict[str, str]
# parse `` and a[x+1]=bar differently
oil_grammar = None # type: Grammar
if mylib.PYTHON:
loader = pyutil.GetResourceLoader()
oil_grammar = pyutil.LoadOilGrammar(loader)
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
pretty_print = True
if len(argv) == 1:
line_reader = reader.FileLineReader(mylib.Stdin(), arena)
src = source.Stdin('') # type: source_t
elif len(argv) == 2:
path = argv[1]
f = mylib.open(path)
line_reader = reader.FileLineReader(f, arena)
src = source.MainFile(path)
elif len(argv) == 3:
if argv[1] == '-c':
# This path is easier to run through GDB
line_reader = reader.StringLineReader(argv[2], arena)
src = source.CFlag()
elif argv[1] == '-n': # For benchmarking, allow osh_parse -n file.txt
path = argv[2]
f = mylib.open(path)
line_reader = reader.FileLineReader(f, arena)
src = source.MainFile(path)
# This is like --ast-format none, which benchmarks/osh-helper.sh passes.
pretty_print = False
else:
raise AssertionError()
else:
raise AssertionError()
arena.PushSource(src)
c_parser = parse_ctx.MakeOshParser(line_reader)
try:
#node = main_loop.ParseWholeFile(c_parser)
node = ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
return 2
assert node is not None
# C++ doesn't have the abbreviations yet (though there are some differences
# like omitting spids)
#tree = node.AbbreviatedTree()
if pretty_print:
tree = node.PrettyTree()
ast_f = fmt.DetectConsoleOutput(mylib.Stdout())
fmt.PrintTree(tree, ast_f)
ast_f.write('\n')
return 0
def InitLexer(s, arena):
"""For tests only."""
line_lexer = lexer.LineLexer('', arena)
line_reader = reader.StringLineReader(s, arena)
lx = lexer.Lexer(line_lexer, line_reader)
return line_reader, lx
def Main(lang, arg_r, environ, login_shell, loader, line_input):
# type: (str, args.Reader, Dict[str, str], bool, pyutil._ResourceLoader, Any) -> int
"""The full shell lifecycle. Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, arg_r: command line arguments
environ: environment
login_shell: Was - on the front?
loader: to get help, version, grammar, etc.
line_input: optional GNU readline
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - shopt -s oil:all
# - Change the prompt in the interactive shell?
# osh-pure:
# - no oil grammar
# - no expression evaluator
# - no interactive shell, or line_input
# - no process.*
# process.{ExternalProgram,Waiter,FdState,JobState,SignalState} -- we want
# to evaluate config files without any of these
# Modules not translated yet: completion, comp_ui, builtin_comp, process
# - word evaluator
# - shouldn't glob? set -o noglob? or hard failure?
# - ~ shouldn't read from the file system
# - I guess it can just be the HOME=HOME?
# Builtin:
# shellvm -c 'echo hi'
# shellvm <<< 'echo hi'
argv0 = arg_r.Peek()
assert argv0 is not None
arg_r.Next()
assert lang in ('osh', 'oil'), lang
try:
attrs = flag_spec.ParseMore('main', arg_r)
except error.Usage as e:
stderr_line('osh usage error: %s', e.msg)
return 2
flag = arg_types.main(attrs.attrs)
arena = alloc.Arena()
errfmt = ui.ErrorFormatter(arena)
help_builtin = builtin_misc.Help(loader, errfmt)
if flag.help:
help_builtin.Run(pure.MakeBuiltinArgv(['%s-usage' % lang]))
return 0
if flag.version:
# OSH version is the only binary in Oil right now, so it's all one version.
pyutil.ShowAppVersion('Oil', loader)
return 0
no_str = None # type: str
debug_stack = [] # type: List[state.DebugFrame]
if arg_r.AtEnd():
dollar0 = argv0
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
# Copy quirky bash behavior.
frame0 = state.DebugFrame(dollar0, 'main', no_str, state.LINE_ZERO, 0,
0)
debug_stack.append(frame0)
# Copy quirky bash behavior.
frame1 = state.DebugFrame(no_str, no_str, no_str, runtime.NO_SPID, 0, 0)
debug_stack.append(frame1)
script_name = arg_r.Peek() # type: Optional[str]
arg_r.Next()
mem = state.Mem(dollar0, arg_r.Rest(), arena, debug_stack)
version_str = pyutil.GetVersion(loader)
state.InitMem(mem, environ, version_str)
builtin_funcs.Init(mem)
procs = {} # type: Dict[str, command__ShFunction]
job_state = process.JobState()
fd_state = process.FdState(errfmt, job_state, mem)
opt_hook = ShellOptHook(line_input)
parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
# TODO: only MutableOpts needs mem, so it's not a true circular dep.
mem.exec_opts = exec_opts # circular dep
if attrs.show_options: # special case: sh -o
mutable_opts.ShowOptions([])
return 0
# Set these BEFORE processing flags, so they can be overridden.
if lang == 'oil':
mutable_opts.SetShoptOption('oil:all', True)
builtin_pure.SetShellOpts(mutable_opts, attrs.opt_changes,
attrs.shopt_changes)
# feedback between runtime and parser
aliases = {} # type: Dict[str, str]
oil_grammar = pyutil.LoadOilGrammar(loader)
if flag.one_pass_parse and not exec_opts.noexec():
raise error.Usage('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
parse_ctx.Init_OnePassParse(flag.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena, parse_opts, aliases,
oil_grammar)
comp_ctx.Init_Trail(trail1)
comp_ctx.Init_OnePassParse(True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
oil_grammar)
hist_ctx.Init_Trail(trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - cmd_ev and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - cmd_ev and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
cmd_deps = cmd_eval.Deps()
cmd_deps.mutable_opts = mutable_opts
# TODO: In general, cmd_deps are shared between the mutually recursive
# evaluators. Some of the four below are only shared between a builtin and
# the CommandEvaluator, so we could put them somewhere else.
cmd_deps.traps = {}
cmd_deps.trap_nodes = [] # TODO: Clear on fork() to avoid duplicates
waiter = process.Waiter(job_state, exec_opts)
my_pid = posix.getpid()
debug_path = ''
debug_dir = environ.get('OSH_DEBUG_DIR')
if flag.debug_file is not None:
# --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = flag.debug_file
elif debug_dir is not None:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if len(debug_path):
# This will be created as an empty file if it doesn't exist, or it could be
# a pipe.
try:
debug_f = util.DebugFile(
fd_state.OpenForWrite(debug_path)) # type: util._DebugFile
except OSError as e:
stderr_line("osh: Couldn't open %r: %s", debug_path,
posix.strerror(e.errno))
return 2
else:
debug_f = util.NullDebugFile()
cmd_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
if mylib.PYTHON:
iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid,
arg_r.argv)
if len(debug_path):
debug_f.log('Writing logs to %r', debug_path)
interp = environ.get('OSH_HIJACK_SHEBANG', '')
search_path = state.SearchPath(mem)
ext_prog = process.ExternalProgram(interp, fd_state, errfmt, debug_f)
splitter = split.SplitContext(mem)
# split() builtin
# TODO: Accept IFS as a named arg? split('a b', IFS=' ')
builtin_funcs.SetGlobalFunc(
mem,
'split',
lambda s, ifs=None: splitter.SplitForWordEval(s, ifs=ifs))
# glob() builtin
# TODO: This is instantiation is duplicated in osh/word_eval.py
globber = glob_.Globber(exec_opts)
builtin_funcs.SetGlobalFunc(mem, 'glob', lambda s: globber.OilFuncCall(s))
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = environ.get('OSH_CRASH_DUMP_DIR', '')
cmd_deps.dumper = dev.CrashDumper(crash_dump_dir)
if flag.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(mylib.Stderr())
comp_lookup = completion.Lookup()
# Various Global State objects to work around readline interfaces
compopt_state = completion.OptionState()
comp_ui_state = comp_ui.State()
prompt_state = comp_ui.PromptState()
dir_stack = state.DirStack()
#
# Initialize builtins that don't depend on evaluators
#
builtins = {} # type: Dict[int, vm._Builtin]
pure.AddPure(builtins, mem, procs, mutable_opts, aliases, search_path,
errfmt)
pure.AddIO(builtins, mem, dir_stack, exec_opts, splitter, errfmt)
AddProcess(builtins, mem, ext_prog, fd_state, job_state, waiter,
search_path, errfmt)
AddOil(builtins, mem, errfmt)
builtins[builtin_i.help] = help_builtin
# Interactive, depend on line_input
builtins[builtin_i.bind] = builtin_lib.Bind(line_input, errfmt)
builtins[builtin_i.history] = builtin_lib.History(line_input,
mylib.Stdout())
#
# Assignment builtins
#
assign_b = {} # type: Dict[int, vm._AssignBuiltin]
new_var = builtin_assign.NewVar(mem, procs, errfmt)
assign_b[builtin_i.declare] = new_var
assign_b[builtin_i.typeset] = new_var
assign_b[builtin_i.local] = new_var
assign_b[builtin_i.export_] = builtin_assign.Export(mem, errfmt)
assign_b[builtin_i.readonly] = builtin_assign.Readonly(mem, errfmt)
#
# Initialize Evaluators
#
arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, parse_ctx, errfmt)
bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, parse_ctx, errfmt)
expr_ev = expr_eval.OilEvaluator(mem, procs, splitter, errfmt)
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)
cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
arena, cmd_deps)
shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
builtins, search_path, ext_prog, waiter,
job_state, fd_state, errfmt)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, mem)
tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev,
trace_f)
# Wire up circular dependencies.
vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
prompt_ev, tracer)
#
# Initialize builtins that depend on evaluators
#
# note: 'printf -v a[i]' and 'unset a[i]' require same deps
builtins[builtin_i.printf] = builtin_printf.Printf(mem, exec_opts,
parse_ctx, arith_ev,
errfmt)
builtins[builtin_i.unset] = builtin_assign.Unset(mem, exec_opts, procs,
parse_ctx, arith_ev,
errfmt)
builtins[builtin_i.eval] = builtin_meta.Eval(parse_ctx, exec_opts, cmd_ev)
source_builtin = builtin_meta.Source(parse_ctx, search_path, cmd_ev,
fd_state, errfmt)
builtins[builtin_i.source] = source_builtin
builtins[builtin_i.dot] = source_builtin
builtins[builtin_i.builtin] = builtin_meta.Builtin(shell_ex, errfmt)
builtins[builtin_i.command] = builtin_meta.Command(shell_ex, procs,
aliases, search_path)
spec_builder = builtin_comp.SpecBuilder(cmd_ev, parse_ctx, word_ev,
splitter, comp_lookup)
complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
builtins[builtin_i.complete] = complete_builtin
builtins[builtin_i.compgen] = builtin_comp.CompGen(spec_builder)
builtins[builtin_i.compopt] = builtin_comp.CompOpt(compopt_state, errfmt)
builtins[builtin_i.compadjust] = builtin_comp.CompAdjust(mem)
# These builtins take blocks, and thus need cmd_ev.
builtins[builtin_i.cd] = builtin_misc.Cd(mem, dir_stack, cmd_ev, errfmt)
builtins[builtin_i.json] = builtin_oil.Json(mem, cmd_ev, errfmt)
sig_state = pyos.SignalState()
sig_state.InitShell()
builtins[builtin_i.trap] = builtin_process.Trap(sig_state, cmd_deps.traps,
cmd_deps.trap_nodes,
parse_ctx, errfmt)
# History evaluation is a no-op if line_input is None.
hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)
if flag.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(flag.c,
arena) # type: reader._Reader
if flag.i: # -c and -i can be combined
mutable_opts.set_interactive()
elif flag.i: # force interactive
arena.PushSource(source.Stdin(' -i'))
line_reader = py_reader.InteractiveLineReader(arena, prompt_ev,
hist_ev, line_input,
prompt_state)
mutable_opts.set_interactive()
else:
if script_name is None:
stdin = mylib.Stdin()
if stdin.isatty():
arena.PushSource(source.Interactive())
line_reader = py_reader.InteractiveLineReader(
arena, prompt_ev, hist_ev, line_input, prompt_state)
mutable_opts.set_interactive()
else:
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
except OSError as e:
stderr_line("osh: Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
c_parser = parse_ctx.MakeOshParser(line_reader)
if exec_opts.interactive():
# bash: 'set -o emacs' is the default only in the interactive shell
mutable_opts.set_emacs()
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = pyos.GetMyHomeDir()
assert home_dir is not None
history_filename = os_path.join(home_dir,
'.config/oil/history_%s' % lang)
if line_input:
# NOTE: We're using a different WordEvaluator here.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
errfmt)
ev.arith_ev = arith_ev
ev.expr_ev = expr_ev
ev.prompt_ev = prompt_ev
ev.CheckCircularDeps()
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
compopt_state, comp_ui_state,
comp_ctx, debug_f)
term_width = 0
if flag.completion_display == 'nice':
try:
term_width = libc.get_terminal_width()
except IOError: # stdin not a terminal
pass
if term_width != 0:
display = comp_ui.NiceDisplay(
term_width, comp_ui_state, prompt_state, debug_f,
line_input) # type: comp_ui._IDisplay
else:
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
comp_ui.InitReadline(line_input, history_filename, root_comp,
display, debug_f)
_InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup)
else: # Without readline module
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
sig_state.InitInteractiveShell(display)
rc_path = flag.rcfile or os_path.join(home_dir,
'.config/oil/%src' % lang)
try:
# NOTE: Should be called AFTER _InitDefaultCompletions.
SourceStartupFile(fd_state, rc_path, lang, parse_ctx, cmd_ev)
except util.UserExit as e:
return e.status
line_reader.Reset() # After sourcing startup file, render $PS1
prompt_plugin = prompt.UserPlugin(mem, parse_ctx, cmd_ev)
try:
status = main_loop.Interactive(flag, cmd_ev, c_parser, display,
prompt_plugin, errfmt)
except util.UserExit as e:
status = e.status
box = [status]
cmd_ev.MaybeRunExitTrap(box)
status = box[0]
return status
if flag.rcfile: # bash doesn't have this warning, but it's useful
stderr_line('osh warning: --rcfile ignored in non-interactive shell')
if exec_opts.noexec():
status = 0
try:
node = main_loop.ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
status = 2
if status == 0:
if flag.parser_mem_dump is not None: # only valid in -n mode
input_path = '/proc/%d/status' % posix.getpid()
pyutil.CopyFile(input_path, flag.parser_mem_dump)
ui.PrintAst(node, flag)
else:
if flag.parser_mem_dump is not None:
raise error.Usage('--parser-mem-dump can only be used with -n')
try:
status = main_loop.Batch(cmd_ev,
c_parser,
arena,
cmd_flags=cmd_eval.IsMainProgram)
except util.UserExit as e:
status = e.status
box = [status]
cmd_ev.MaybeRunExitTrap(box)
status = box[0]
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if flag.runtime_mem_dump is not None:
input_path = '/proc/%d/status' % posix.getpid()
pyutil.CopyFile(input_path, flag.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def main(argv):
# type: (List[str]) -> int
arena = alloc.Arena()
dollar0 = argv[0]
debug_stack = [] # type: List[state.DebugFrame]
mem = state.Mem(dollar0, argv, arena, debug_stack)
opt_hook = state.OptHook()
parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
# Dummy value; not respecting aliases!
aliases = {} # type: Dict[str, str]
# parse `` and a[x+1]=bar differently
state.SetGlobalString(mem, 'SHELLOPTS', '')
oil_grammar = None # type: Grammar
if mylib.PYTHON:
loader = pyutil.GetResourceLoader()
oil_grammar = meta.LoadOilGrammar(loader)
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
argv = argv[1:] # remove binary name
i, flag_a, flag_c, flag_n = Parse(argv)
argv = argv[i:] # truncate
if flag_c:
# This path is easier to run through GDB
line_reader = reader.StringLineReader(flag_c, arena)
src = source.CFlag() # type: source_t
elif len(argv) == 0:
line_reader = reader.FileLineReader(mylib.Stdin(), arena)
src = source.Stdin('')
elif len(argv) == 1:
path = argv[0]
f = mylib.open(path)
line_reader = reader.FileLineReader(f, arena)
src = source.MainFile(path)
else:
raise AssertionError(argv)
arena.PushSource(src)
c_parser = parse_ctx.MakeOshParser(line_reader)
# C++ doesn't have the abbreviations yet (though there are some differences
# like omitting spids)
#tree = node.AbbreviatedTree()
if flag_n:
try:
node = main_loop.ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
return 2
assert node is not None
if flag_a:
tree = node.PrettyTree()
ast_f = fmt.DetectConsoleOutput(mylib.Stdout())
fmt.PrintTree(tree, ast_f)
ast_f.write('\n')
return 0
# New osh_eval.py instantiations
errfmt = ui.ErrorFormatter(arena)
splitter = split.SplitContext(mem)
arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, parse_ctx, errfmt)
bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, parse_ctx, errfmt)
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, splitter, errfmt)
arith_ev.word_ev = word_ev
word_ev.arith_ev = arith_ev
procs = {} # type: Dict[str, command__ShFunction]
assign_builtins = {} # type: Dict[int, _AssignBuiltin]
new_var = builtin_assign.NewVar(mem, procs, errfmt)
assign_builtins[builtin_i.declare] = new_var
assign_builtins[builtin_i.typeset] = new_var
assign_builtins[builtin_i.local] = new_var
#assign_builtins = {
# # ShAssignment (which are pure)
# builtin_i.declare: new_var,
# builtin_i.typeset: new_var,
# builtin_i.local: new_var,
# builtin_i.export_: builtin_assign.Export(mem, errfmt),
# builtin_i.readonly: builtin_assign.Readonly(mem, errfmt),
#}
cmd_deps = cmd_eval.Deps()
cmd_deps.mutable_opts = mutable_opts
cmd_deps.traps = {}
cmd_deps.trap_nodes = [] # TODO: Clear on fork() to avoid duplicates
cmd_deps.dumper = dev.CrashDumper('')
builtins = {} # type: Dict[int, _Builtin]
builtins[builtin_i.echo] = Echo()
builtins[builtin_i.shopt] = Shopt(mutable_opts)
builtins[builtin_i.set] = Set(mutable_opts)
ex = NullExecutor(builtins)
trace_f = util.DebugFile(mylib.Stderr())
tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, word_ev, trace_f)
cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
assign_builtins, arena, cmd_deps)
# vm.InitCircularDeps
cmd_ev.arith_ev = arith_ev
cmd_ev.bool_ev = bool_ev
cmd_ev.word_ev = word_ev
cmd_ev.tracer = tracer
cmd_ev.shell_ex = ex
bool_ev.word_ev = word_ev
status = main_loop.Batch(cmd_ev, c_parser, arena, is_main=True)
return status
def ShellMain(lang, argv0, argv, login_shell):
"""Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox.
login_shell: Was - on the front?
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - the parser and executor
# - Change the prompt in the interactive shell?
assert lang in ('osh', 'oil'), lang
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.usage('osh usage error: %s', e)
return 2
if opts.help:
loader = util.GetResourceLoader()
builtin.Help(['%s-usage' % lang], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
# TODO: This should be in interactive mode only?
builtin.RegisterSigIntHandler()
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
pool = alloc.Pool()
arena = pool.NewArena()
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
funcs = {}
fd_state = process.FdState()
exec_opts = state.ExecOpts(mem, readline)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
aliases = {} # feedback between runtime and parser
parse_ctx = parse_lib.ParseContext(arena, aliases) # For main_loop
# Three ParseContext instances SHARE aliases. TODO: Complete aliases.
comp_arena = pool.NewArena()
comp_arena.PushSource('<completion>')
trail1 = parse_lib.Trail()
comp_ctx = parse_lib.ParseContext(comp_arena, aliases, trail=trail1)
hist_arena = pool.NewArena()
hist_arena.PushSource('<history>')
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - ex and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - ex and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
exec_deps = cmd_exec.Deps()
if opts.debug_file:
debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w'))
else:
debug_f = util.NullDebugFile()
exec_deps.debug_f = debug_f
debug_f.log('Debug file is %s', opts.debug_file)
splitter = split.SplitContext(mem)
exec_deps.splitter = splitter
# Controlled by env variable, flag, or hook?
exec_deps.dumper = dev.CrashDumper(
posix.environ.get('OSH_CRASH_DUMP_DIR', ''))
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
exec_deps.trace_f = trace_f
# TODO: Separate comp_state and comp_lookup.
comp_state = completion.State()
comp_lookup = completion.Lookup()
builtins = { # Lookup
builtin_e.HISTORY: builtin.History(readline),
builtin_e.COMPOPT: builtin_comp.CompOpt(comp_state),
builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
}
ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
parse_ctx, exec_deps)
exec_deps.ex = ex
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
exec_deps.word_ev = word_ev
arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.arith_ev = arith_ev
word_ev.arith_ev = arith_ev # Another circular dependency
bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.bool_ev = bool_ev
tracer = cmd_exec.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
exec_deps.tracer = tracer
# HACK for circular deps
ex.word_ev = word_ev
ex.arith_ev = arith_ev
ex.bool_ev = bool_ev
ex.tracer = tracer
spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter)
# Add some builtins that depend on the executor!
complete_builtin = builtin_comp.Complete(spec_builder,
comp_lookup) # used later
builtins[builtin_e.COMPLETE] = complete_builtin
builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)
if lang == 'oil':
# The Oil executor wraps an OSH executor? It needs to be able to source
# it.
ex = oil_cmd_exec.OilExecutor(ex)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = ui.PromptEvaluator(lang, arena, parse_ctx, ex, mem)
exec_deps.prompt_ev = prompt_ev
word_ev.prompt_ev = prompt_ev # HACK for circular deps
# History evaluation is a no-op if readline is None.
hist_ev = reader.HistoryEvaluator(readline, hist_ctx, debug_f)
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = mem.GetVar('HOME')
assert home_dir.tag == value_e.Str, home_dir
rc_path = opts.rcfile or os_path.join(home_dir.s, '.config/oil',
lang + 'rc')
history_filename = os_path.join(home_dir.s, '.config/oil',
'history_' + lang)
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
# interactive shell only
line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
# interactive shell only
line_reader = reader.InteractiveLineReader(
arena, prompt_ev, hist_ev)
exec_opts.interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(script_name)
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
if lang == 'osh':
c_parser = parse_ctx.MakeOshParser(line_reader)
else:
c_parser = parse_ctx.MakeOilParser(line_reader)
if exec_opts.interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if readline:
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
arena)
progress_f = ui.StatusLine()
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
comp_state, comp_ctx,
progress_f, debug_f)
_InitReadline(readline, history_filename, root_comp, debug_f)
_InitDefaultCompletions(ex, complete_builtin, comp_lookup)
# NOTE: Call this AFTER _InitDefaultCompletions.
SourceStartupFile(rc_path, lang, parse_ctx, ex)
return main_loop.Interactive(opts, ex, c_parser, arena)
# TODO: Remove this after removing it from benchmarks/osh-runtime. It's no
# longer relevant with main_loop.
if opts.parser_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
nodes_out = [] if exec_opts.noexec else None
_tlog('Execute(node)')
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
# Only print nodes if the whole parse succeeded.
if nodes_out is not None and status == 0:
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: This doesn't cause any spec tests to fail, but it could.
if posix.environ.get('ASDL_TYPE_CHECK'):
log('NOTE: Performed %d ASDL_TYPE_CHECKs.', runtime.NUM_TYPE_CHECKS)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def ShellMain(lang, argv0, argv, login_shell):
"""Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox.
login_shell: Was - on the front?
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - the parser and executor
# - Change the prompt in the interactive shell?
assert lang in ('osh', 'oil'), lang
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.usage('osh usage error: %s', e)
return 2
# NOTE: This has a side effect of deleting _OVM_* from the environment!
# TODO: Thread this throughout the program, and get rid of the global
# variable in core/util.py. Rename to InitResourceLaoder(). It's now only
# used for the 'help' builtin and --version.
loader = pyutil.GetResourceLoader()
if opts.help:
builtin.Help(['%s-usage' % lang], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
arena = alloc.Arena()
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
funcs = {}
fd_state = process.FdState()
exec_opts = state.ExecOpts(mem, line_input)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
aliases = {} # feedback between runtime and parser
if opts.one_pass_parse and not exec_opts.noexec:
raise args.UsageError('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena,
aliases,
one_pass_parse=opts.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena,
aliases,
trail=trail1,
one_pass_parse=True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena, aliases, trail=trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - ex and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - ex and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
exec_deps = cmd_exec.Deps()
my_pid = posix.getpid()
debug_path = ''
debug_dir = posix.environ.get('OSH_DEBUG_DIR')
if opts.debug_file: # --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = opts.debug_file
elif debug_dir:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if debug_path:
# This will be created as an empty file if it doesn't exist, or it could be
# a pipe.
try:
debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
except OSError as e:
util.error("Couldn't open %r: %s", debug_path,
posix.strerror(e.errno))
return 2
else:
debug_f = util.NullDebugFile()
exec_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
if debug_path:
debug_f.log('Writing logs to %r', debug_path)
interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
exec_deps.ext_prog = process.ExternalProgram(interp, fd_state, debug_f)
splitter = split.SplitContext(mem)
exec_deps.splitter = splitter
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
exec_deps.dumper = dev.CrashDumper(crash_dump_dir)
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
exec_deps.trace_f = trace_f
comp_lookup = completion.Lookup()
#
# Various Global State objects to work around readline interfaces
#
compopt_state = completion.OptionState()
comp_ui_state = comp_ui.State()
prompt_state = comp_ui.PromptState()
builtins = { # Lookup
builtin_e.HISTORY: builtin.History(line_input),
builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state),
builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
}
ex = cmd_exec.Executor(mem, fd_state, funcs, builtins, exec_opts,
parse_ctx, exec_deps)
exec_deps.ex = ex
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
exec_deps.word_ev = word_ev
arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.arith_ev = arith_ev
word_ev.arith_ev = arith_ev # Another circular dependency
bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, word_ev, arena)
exec_deps.bool_ev = bool_ev
tracer = dev.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
exec_deps.tracer = tracer
# HACK for circular deps
ex.word_ev = word_ev
ex.arith_ev = arith_ev
ex.bool_ev = bool_ev
ex.tracer = tracer
spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
comp_lookup)
# Add some builtins that depend on the executor!
complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
builtins[builtin_e.COMPLETE] = complete_builtin
builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)
if lang == 'oil':
# The Oil executor wraps an OSH executor? It needs to be able to source
# it.
ex = oil_cmd_exec.OilExecutor(ex)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, ex, mem)
exec_deps.prompt_ev = prompt_ev
word_ev.prompt_ev = prompt_ev # HACK for circular deps
# History evaluation is a no-op if line_input is None.
hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = process.GetHomeDir()
assert home_dir is not None
rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil', lang + 'rc')
history_filename = os_path.join(home_dir, '.config/oil', 'history_' + lang)
if opts.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource(source.Stdin(' -i'))
# interactive shell only
line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
line_input, prompt_state)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource(source.Interactive())
# interactive shell only
line_reader = reader.InteractiveLineReader(
arena, prompt_ev, hist_ev, line_input, prompt_state)
exec_opts.interactive = True
else:
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
if lang == 'osh':
c_parser = parse_ctx.MakeOshParser(line_reader)
else:
c_parser = parse_ctx.MakeOilParser(line_reader)
# NOTE: SIGINT is temporarily enabled during readline() by
# frontend/reader.py.
# It's treated differently than SIGQUIT and SIGTSTP because Python handles it
# with KeyboardInterrupt. We don't want KeyboardInterrupt at arbitrary
# points in a non-interactive shell. (e.g. osh -c 'sleep 5' then Ctrl-C)
signal.signal(signal.SIGINT, signal.SIG_IGN)
if exec_opts.interactive:
if line_input:
# NOTE: We're using a different WordEvaluator here.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
arena)
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
compopt_state, comp_ui_state,
comp_ctx, debug_f)
term_width = 0
if opts.completion_display == 'nice':
try:
term_width = libc.get_terminal_width()
except IOError: # stdin not a terminal
pass
if term_width != 0:
display = comp_ui.NiceDisplay(term_width, comp_ui_state,
prompt_state, debug_f,
line_input)
else:
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
_InitReadline(line_input, history_filename, root_comp, display,
debug_f)
_InitDefaultCompletions(ex, complete_builtin, comp_lookup)
else: # Without readline module
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
# The shell itself should ignore Ctrl-\.
signal.signal(signal.SIGQUIT, signal.SIG_IGN)
# This prevents Ctrl-Z from suspending OSH in interactive mode. But we're
# not getting notification via wait() that the child stopped?
signal.signal(signal.SIGTSTP, signal.SIG_IGN)
# Register a callback to receive terminal width changes.
signal.signal(signal.SIGWINCH, lambda x, y: display.OnWindowChange())
# NOTE: Call this AFTER _InitDefaultCompletions.
SourceStartupFile(rc_path, lang, parse_ctx, ex)
line_reader.Reset() # After sourcing startup file, render $PS1
return main_loop.Interactive(opts, ex, c_parser, display, arena)
nodes_out = [] if exec_opts.noexec else None
if nodes_out is None and opts.parser_mem_dump:
raise args.UsageError('--parser-mem-dump can only be used with -n')
_tlog('Execute(node)')
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
# Only print nodes if the whole parse succeeded.
if nodes_out is not None and status == 0:
if opts.parser_mem_dump: # only valid in -n mode
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def Matches(self, comp):
# type: (Api) -> Iterator[Union[Iterator, Iterator[str]]]
"""
Args:
comp: Callback args from readline. Readline uses set_completer_delims to
tokenize the string.
Returns a list of matches relative to readline's completion_delims.
We have to post-process the output of various completers.
"""
arena = self.parse_ctx.arena # Used by inner functions
# Pass the original line "out of band" to the completion callback.
line_until_tab = comp.line[:comp.end]
self.comp_ui_state.line_until_tab = line_until_tab
self.parse_ctx.trail.Clear()
line_reader = reader.StringLineReader(line_until_tab,
self.parse_ctx.arena)
c_parser = self.parse_ctx.MakeOshParser(line_reader,
emit_comp_dummy=True)
# We want the output from parse_ctx, so we don't use the return value.
try:
c_parser.ParseLogicalLine()
except error.Parse as e:
# e.g. 'ls | ' will not parse. Now inspect the parser state!
pass
debug_f = self.debug_f
trail = self.parse_ctx.trail
if 1:
trail.PrintDebugString(debug_f)
#
# First try completing the shell language itself.
#
# NOTE: We get Eof_Real in the command state, but not in the middle of a
# BracedVarSub. This is due to the difference between the CommandParser
# and WordParser.
tokens = trail.tokens
last = -1
if tokens[-1].id == Id.Eof_Real:
last -= 1 # ignore it
try:
t1 = tokens[last]
except IndexError:
t1 = None
try:
t2 = tokens[last - 1]
except IndexError:
t2 = None
debug_f.log('line: %r', comp.line)
debug_f.log('rl_slice from byte %d to %d: %r', comp.begin, comp.end,
comp.line[comp.begin:comp.end])
debug_f.log('t1 %s', t1)
debug_f.log('t2 %s', t2)
# Each of the 'yield' statements below returns a fully-completed line, to
# appease the readline library. The root cause of this dance: If there's
# one candidate, readline is responsible for redrawing the input line. OSH
# only displays candidates and never redraws the input line.
def _TokenStart(tok):
# type: (Token) -> int
span = arena.GetLineSpan(tok.span_id)
return span.col
if t2: # We always have t1?
# echo $
if IsDollar(t2) and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $
for name in self.mem.VarNames():
yield line_until_tab + name # no need to quote var names
return
# echo ${
if t2.id == Id.Left_DollarBrace and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(
t2) + 2 # 2 for ${
for name in self.mem.VarNames():
yield line_until_tab + name # no need to quote var names
return
# echo $P
if t2.id == Id.VSub_DollarName and IsDummy(t1):
# Example: ${undef:-$P
# readline splits at ':' so we have to prepend '-$' to every completed
# variable name.
self.comp_ui_state.display_pos = _TokenStart(t2) + 1 # 1 for $
to_complete = t2.val[1:]
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[
n:] # no need to quote var names
return
# echo ${P
if t2.id == Id.VSub_Name and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) # no offset
to_complete = t2.val
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[
n:] # no need to quote var names
return
# echo $(( VAR
if t2.id == Id.Lit_ArithVarLike and IsDummy(t1):
self.comp_ui_state.display_pos = _TokenStart(t2) # no offset
to_complete = t2.val
n = len(to_complete)
for name in self.mem.VarNames():
if name.startswith(to_complete):
yield line_until_tab + name[
n:] # no need to quote var names
return
if trail.words:
# echo ~<TAB>
# echo ~a<TAB> $(home dirs)
# This must be done at a word level, and TildeDetectAll() does NOT help
# here, because they don't have trailing slashes yet! We can't do it on
# tokens, because otherwise f~a will complete. Looking at word_part is
# EXACTLY what we want.
parts = trail.words[-1].parts
if (len(parts) == 2 and parts[0].tag_() == word_part_e.Literal
and parts[1].tag_() == word_part_e.Literal
and parts[0].id == Id.Lit_TildeLike
and parts[1].id == Id.Lit_CompDummy):
t2 = parts[0]
# +1 for ~
self.comp_ui_state.display_pos = _TokenStart(parts[0]) + 1
to_complete = t2.val[1:]
n = len(to_complete)
for u in pwd.getpwall(): # catch errors?
name = u.pw_name
if name.startswith(to_complete):
yield line_until_tab + ShellQuoteB(name[n:]) + '/'
return
# echo hi > f<TAB> (complete redirect arg)
if trail.redirects:
r = trail.redirects[-1]
# Only complete 'echo >', but not 'echo >&' or 'cat <<'
# TODO: Don't complete <<< 'h'
if (r.arg.tag_() == redir_param_e.Word
and consts.RedirArgType(r.op.id) == redir_arg_type_e.Path):
arg_word = r.arg
if WordEndsWithCompDummy(arg_word):
debug_f.log('Completing redirect arg')
try:
val = self.word_ev.EvalWordToString(r.arg)
except error.FatalRuntime as e:
debug_f.log('Error evaluating redirect word: %s', e)
return
if val.tag_() != value_e.Str:
debug_f.log("Didn't get a string from redir arg")
return
span_id = word_.LeftMostSpanForWord(arg_word)
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
comp.Update(
to_complete=val.s) # FileSystemAction uses only this
n = len(val.s)
action = FileSystemAction(add_slash=True)
for name in action.Matches(comp):
yield line_until_tab + ShellQuoteB(name[n:])
return
#
# We're not completing the shell language. Delegate to user-defined
# completion for external tools.
#
# Set below, and set on retries.
base_opts = None
user_spec = None
# Used on retries.
partial_argv = []
num_partial = -1
first = None
if trail.words:
# Now check if we're completing a word!
if WordEndsWithCompDummy(trail.words[-1]):
debug_f.log('Completing words')
#
# It didn't look like we need to complete var names, tilde, redirects,
# etc. Now try partial_argv, which may involve invoking PLUGINS.
# needed to complete paths with ~
words2 = word_.TildeDetectAll(trail.words)
if 0:
debug_f.log('After tilde detection')
for w in words2:
print(w, file=debug_f)
if 0:
debug_f.log('words2:')
for w2 in words2:
debug_f.log(' %s', w2)
for w in words2:
try:
# TODO:
# - Should we call EvalWordSequence? But turn globbing off? It
# can do splitting and such.
# - We could have a variant to eval TildeSub to ~ ?
val = self.word_ev.EvalWordToString(w)
except error.FatalRuntime:
# Why would it fail?
continue
if val.tag_() == value_e.Str:
partial_argv.append(val.s)
else:
pass
debug_f.log('partial_argv: %s', partial_argv)
num_partial = len(partial_argv)
first = partial_argv[0]
alias_first = None
debug_f.log('alias_words: %s', trail.alias_words)
if trail.alias_words:
w = trail.alias_words[0]
try:
val = self.word_ev.EvalWordToString(w)
except error.FatalRuntime:
pass
alias_first = val.s
debug_f.log('alias_first: %s', alias_first)
if num_partial == 0: # should never happen because of Lit_CompDummy
raise AssertionError()
elif num_partial == 1:
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
# Display/replace since the beginning of the first word. Note: this
# is non-zero in the case of
# echo $(gr and
# echo `gr
span_id = word_.LeftMostSpanForWord(trail.words[0])
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
self.debug_f.log('** DISPLAY_POS = %d',
self.comp_ui_state.display_pos)
else:
base_opts, user_spec = self.comp_lookup.GetSpecForName(
first)
if not user_spec and alias_first:
base_opts, user_spec = self.comp_lookup.GetSpecForName(
alias_first)
if user_spec:
# Pass the aliased command to the user-defined function, and use
# it for retries.
first = alias_first
if not user_spec:
base_opts, user_spec = self.comp_lookup.GetFallback()
# Display since the beginning
span_id = word_.LeftMostSpanForWord(trail.words[-1])
span = arena.GetLineSpan(span_id)
self.comp_ui_state.display_pos = span.col
self.debug_f.log('words[-1]: %r', trail.words[-1])
self.debug_f.log('display_pos %d',
self.comp_ui_state.display_pos)
# Update the API for user-defined functions.
index = len(
partial_argv) - 1 # COMP_CWORD is -1 when it's empty
prev = '' if index == 0 else partial_argv[index - 1]
comp.Update(first=first,
to_complete=partial_argv[-1],
prev=prev,
index=index,
partial_argv=partial_argv)
# This happens in the case of [[ and ((, or a syntax error like 'echo < >'.
if not user_spec:
debug_f.log("Didn't find anything to complete")
return
# Reset it back to what was registered. User-defined functions can mutate
# it.
dynamic_opts = {}
self.compopt_state.dynamic_opts = dynamic_opts
self.compopt_state.currently_completing = True
try:
done = False
while not done:
try:
for candidate in self._PostProcess(base_opts, dynamic_opts,
user_spec, comp):
yield candidate
except _RetryCompletion as e:
debug_f.log('Got 124, trying again ...')
# Get another user_spec. The ShellFuncAction may have 'sourced' code
# and run 'complete' to mutate comp_lookup, and we want to get that
# new entry.
if num_partial == 0:
raise AssertionError()
elif num_partial == 1:
base_opts, user_spec = self.comp_lookup.GetFirstSpec()
else:
# (already processed alias_first)
base_opts, user_spec = self.comp_lookup.GetSpecForName(
first)
if not user_spec:
base_opts, user_spec = self.comp_lookup.GetFallback(
)
else:
done = True # exhausted candidates without getting a retry
finally:
self.compopt_state.currently_completing = False
def MakeOilLexer(code_str, arena):
arena.PushSource(source.MainFile('pgen2_main'))
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer('', arena)
lex = lexer.Lexer(line_lexer, line_reader)
return lex
def Run(self, cmd_val):
# type: (cmd_value__Argv) -> int
"""
printf: printf [-v var] format [argument ...]
"""
attrs, arg_r = flag_spec.ParseCmdVal('printf', cmd_val)
arg = arg_types.printf(attrs.attrs)
fmt, fmt_spid = arg_r.ReadRequired2('requires a format string')
varargs, spids = arg_r.Rest2()
#log('fmt %s', fmt)
#log('vals %s', vals)
arena = self.parse_ctx.arena
if fmt in self.parse_cache:
parts = self.parse_cache[fmt]
else:
line_reader = reader.StringLineReader(fmt, arena)
# TODO: Make public
lexer = self.parse_ctx._MakeLexer(line_reader)
parser = _FormatStringParser(lexer)
with alloc.ctx_Location(arena, source.ArgvWord(fmt_spid)):
try:
parts = parser.Parse()
except error.Parse as e:
self.errfmt.PrettyPrintError(e)
return 2 # parse error
self.parse_cache[fmt] = parts
if 0:
print()
for part in parts:
part.PrettyPrint()
print()
out = [] # type: List[str]
arg_index = 0
num_args = len(varargs)
backslash_c = False
while True:
for part in parts:
UP_part = part
if part.tag_() == printf_part_e.Literal:
part = cast(printf_part__Literal, UP_part)
token = part.token
if token.id == Id.Format_EscapedPercent:
s = '%'
else:
s = word_compile.EvalCStringToken(token)
out.append(s)
elif part.tag_() == printf_part_e.Percent:
part = cast(printf_part__Percent, UP_part)
flags = [] # type: List[str]
if len(part.flags) > 0:
for flag_token in part.flags:
flags.append(flag_token.val)
width = -1 # nonexistent
if part.width:
if part.width.id in (Id.Format_Num, Id.Format_Zero):
width_str = part.width.val
width_spid = part.width.span_id
elif part.width.id == Id.Format_Star:
if arg_index < num_args:
width_str = varargs[arg_index]
width_spid = spids[arg_index]
arg_index += 1
else:
width_str = '' # invalid
width_spid = runtime.NO_SPID
else:
raise AssertionError()
try:
width = int(width_str)
except ValueError:
if width_spid == runtime.NO_SPID:
width_spid = part.width.span_id
self.errfmt.Print_("printf got invalid width %r" %
width_str,
span_id=width_spid)
return 1
precision = -1 # nonexistent
if part.precision:
if part.precision.id == Id.Format_Dot:
precision_str = '0'
precision_spid = part.precision.span_id
elif part.precision.id in (Id.Format_Num,
Id.Format_Zero):
precision_str = part.precision.val
precision_spid = part.precision.span_id
elif part.precision.id == Id.Format_Star:
if arg_index < num_args:
precision_str = varargs[arg_index]
precision_spid = spids[arg_index]
arg_index += 1
else:
precision_str = ''
precision_spid = runtime.NO_SPID
else:
raise AssertionError()
try:
precision = int(precision_str)
except ValueError:
if precision_spid == runtime.NO_SPID:
precision_spid = part.precision.span_id
self.errfmt.Print_(
'printf got invalid precision %r' %
precision_str,
span_id=precision_spid)
return 1
#log('index=%d n=%d', arg_index, num_args)
if arg_index < num_args:
s = varargs[arg_index]
word_spid = spids[arg_index]
arg_index += 1
else:
s = ''
word_spid = runtime.NO_SPID
typ = part.type.val
if typ == 's':
if precision >= 0:
s = s[:precision] # truncate
elif typ == 'q':
s = qsn.maybe_shell_encode(s)
elif typ == 'b':
# Process just like echo -e, except \c handling is simpler.
c_parts = [] # type: List[str]
lex = match.EchoLexer(s)
while True:
id_, tok_val = lex.Next()
if id_ == Id.Eol_Tok: # Note: This is really a NUL terminator
break
# TODO: add span_id from argv
tok = Token(id_, runtime.NO_SPID, tok_val)
p = word_compile.EvalCStringToken(tok)
# Unusual behavior: '\c' aborts processing!
if p is None:
backslash_c = True
break
c_parts.append(p)
s = ''.join(c_parts)
elif typ in 'diouxX' or part.type.id == Id.Format_Time:
try:
d = int(s)
except ValueError:
if len(s) >= 1 and s[0] in '\'"':
# TODO: utf-8 decode s[1:] to be more correct. Probably
# depends on issue #366, a utf-8 library.
# Note: len(s) == 1 means there is a NUL (0) after the quote..
d = ord(s[1]) if len(s) >= 2 else 0
elif part.type.id == Id.Format_Time and len(
s) == 0 and word_spid == runtime.NO_SPID:
# Note: No argument means -1 for %(...)T as in Bash Reference
# Manual 4.2 "If no argument is specified, conversion behaves
# as if -1 had been given."
d = -1
else:
if word_spid == runtime.NO_SPID:
# Blame the format string
blame_spid = part.type.span_id
else:
blame_spid = word_spid
self.errfmt.Print_(
'printf expected an integer, got %r' % s,
span_id=blame_spid)
return 1
if typ in 'di':
s = str(d)
elif typ in 'ouxX':
if d < 0:
e_die(
"Can't format negative number %d with %%%s",
d,
typ,
span_id=part.type.span_id)
if typ == 'u':
s = str(d)
elif typ == 'o':
s = mylib.octal(d)
elif typ == 'x':
s = mylib.hex_lower(d)
elif typ == 'X':
s = mylib.hex_upper(d)
elif part.type.id == Id.Format_Time:
# %(...)T
# Initialize timezone:
# `localtime' uses the current timezone information initialized
# by `tzset'. The function `tzset' refers to the environment
# variable `TZ'. When the exported variable `TZ' is present,
# its value should be reflected in the real environment
# variable `TZ' before call of `tzset'.
#
# Note: unlike LANG, TZ doesn't seem to change behavior if it's
# not exported.
#
# TODO: In Oil, provide an API that doesn't rely on libc's
# global state.
tzcell = self.mem.GetCell('TZ')
if tzcell and tzcell.exported and tzcell.val.tag_(
) == value_e.Str:
tzval = cast(value__Str, tzcell.val)
posix.putenv('TZ', tzval.s)
time_.tzset()
# Handle special values:
# User can specify two special values -1 and -2 as in Bash
# Reference Manual 4.2: "Two special argument values may be
# used: -1 represents the current time, and -2 represents the
# time the shell was invoked." from
# https://www.gnu.org/software/bash/manual/html_node/Bash-Builtins.html#index-printf
if d == -1: # the current time
ts = time_.time()
elif d == -2: # the shell start time
ts = self.shell_start_time
else:
ts = d
s = time_.strftime(typ[1:-2], time_.localtime(ts))
if precision >= 0:
s = s[:precision] # truncate
else:
raise AssertionError()
else:
raise AssertionError()
if width >= 0:
if len(flags):
if '-' in flags:
s = s.ljust(width, ' ')
elif '0' in flags:
s = s.rjust(width, '0')
else:
pass
else:
s = s.rjust(width, ' ')
out.append(s)
else:
raise AssertionError()
if backslash_c: # 'printf %b a\cb xx' - \c terminates processing!
break
if arg_index >= num_args:
break
# Otherwise there are more args. So cycle through the loop once more to
# implement the 'arg recycling' behavior.
result = ''.join(out)
if arg.v is not None:
# TODO: get the span_id for arg.v!
v_spid = runtime.NO_SPID
arena = self.parse_ctx.arena
a_parser = self.parse_ctx.MakeArithParser(arg.v)
with alloc.ctx_Location(arena, source.ArgvWord(v_spid)):
try:
anode = a_parser.Parse()
except error.Parse as e:
ui.PrettyPrintError(e, arena) # show parse error
e_usage('Invalid -v expression', span_id=v_spid)
lval = self.arith_ev.EvalArithLhs(anode, v_spid)
if not self.exec_opts.eval_unsafe_arith(
) and lval.tag_() != lvalue_e.Named:
e_usage(
'-v expected a variable name. shopt -s eval_unsafe_arith allows expressions',
span_id=v_spid)
state.SetRef(self.mem, lval, value.Str(result))
else:
mylib.Stdout().write(result)
return 0
def MakeWordParserForPlugin(self, code_str):
# type: (str) -> WordParser
"""For $PS1, $PS4, etc."""
line_reader = reader.StringLineReader(code_str, self.arena)
lx = self._MakeLexer(line_reader)
return word_parse.WordParser(self, lx, line_reader)
def ShellMain(lang, argv0, argv, login_shell):
"""Used by bin/osh and bin/oil.
Args:
lang: 'osh' or 'oil'
argv0, argv: So we can also invoke bin/osh as 'oil.ovm osh'. Like busybox.
login_shell: Was - on the front?
"""
# Differences between osh and oil:
# - --help? I guess Oil has a SUPERSET of OSH options.
# - oshrc vs oilrc
# - the parser and executor
# - Change the prompt in the interactive shell?
assert lang in ('osh', 'oil'), lang
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.Stderr('osh usage error: %s', e.msg)
return 2
# NOTE: This has a side effect of deleting _OVM_* from the environment!
# TODO: Thread this throughout the program, and get rid of the global
# variable in core/util.py. Rename to InitResourceLaoder(). It's now only
# used for the 'help' builtin and --version.
loader = pyutil.GetResourceLoader()
if opts.help:
builtin.Help(['%s-usage' % lang], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
arena = alloc.Arena()
errfmt = ui.ErrorFormatter(arena)
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
builtin_funcs.Init(mem)
procs = {}
job_state = process.JobState()
fd_state = process.FdState(errfmt, job_state)
parse_opts = parse_lib.OilParseOptions()
exec_opts = state.ExecOpts(mem, parse_opts, line_input)
if opts.show_options: # special case: sh -o
exec_opts.ShowOptions([])
return 0
builtin_pure.SetExecOpts(exec_opts, opts.opt_changes, opts.shopt_changes)
aliases = {} # feedback between runtime and parser
oil_grammar = meta.LoadOilGrammar(loader)
if opts.one_pass_parse and not exec_opts.noexec:
raise args.UsageError('--one-pass-parse requires noexec (-n)')
parse_ctx = parse_lib.ParseContext(arena,
parse_opts,
aliases,
oil_grammar,
one_pass_parse=opts.one_pass_parse)
# Three ParseContext instances SHARE aliases.
comp_arena = alloc.Arena()
comp_arena.PushSource(source.Unused('completion'))
trail1 = parse_lib.Trail()
# one_pass_parse needs to be turned on to complete inside backticks. TODO:
# fix the issue where ` gets erased because it's not part of
# set_completer_delims().
comp_ctx = parse_lib.ParseContext(comp_arena,
parse_opts,
aliases,
oil_grammar,
trail=trail1,
one_pass_parse=True)
hist_arena = alloc.Arena()
hist_arena.PushSource(source.Unused('history'))
trail2 = parse_lib.Trail()
hist_ctx = parse_lib.ParseContext(hist_arena,
parse_opts,
aliases,
oil_grammar,
trail=trail2)
# Deps helps manages dependencies. These dependencies are circular:
# - ex and word_ev, arith_ev -- for command sub, arith sub
# - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
# - ex and builtins (which execute code, like eval)
# - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
exec_deps = cmd_exec.Deps()
# TODO: In general, exec_deps are shared between the mutually recursive
# evaluators. Some of the four below are only shared between a builtin and
# the Executor, so we could put them somewhere else.
exec_deps.traps = {}
exec_deps.trap_nodes = [] # TODO: Clear on fork() to avoid duplicates
exec_deps.job_state = job_state
# note: exec_opts.interactive set later
exec_deps.waiter = process.Waiter(job_state, exec_opts)
exec_deps.errfmt = errfmt
my_pid = posix.getpid()
debug_path = ''
debug_dir = posix.environ.get('OSH_DEBUG_DIR')
if opts.debug_file: # --debug-file takes precedence over OSH_DEBUG_DIR
debug_path = opts.debug_file
elif debug_dir:
debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
if debug_path:
# This will be created as an empty file if it doesn't exist, or it could be
# a pipe.
try:
debug_f = util.DebugFile(fd_state.Open(debug_path, mode='w'))
except OSError as e:
ui.Stderr("osh: Couldn't open %r: %s", debug_path,
posix.strerror(e.errno))
return 2
else:
debug_f = util.NullDebugFile()
exec_deps.debug_f = debug_f
# Not using datetime for dependency reasons. TODO: maybe show the date at
# the beginning of the log, and then only show time afterward? To save
# space, and make space for microseconds. (datetime supports microseconds
# but time.strftime doesn't).
iso_stamp = time.strftime("%Y-%m-%d %H:%M:%S")
debug_f.log('%s [%d] OSH started with argv %s', iso_stamp, my_pid, argv)
if debug_path:
debug_f.log('Writing logs to %r', debug_path)
interp = posix.environ.get('OSH_HIJACK_SHEBANG', '')
exec_deps.search_path = state.SearchPath(mem)
exec_deps.ext_prog = process.ExternalProgram(interp, fd_state,
exec_deps.search_path, errfmt,
debug_f)
splitter = split.SplitContext(mem)
exec_deps.splitter = splitter
# split() builtin
builtin_funcs.SetGlobalFunc(mem, 'split',
lambda s: splitter.SplitForWordEval(s))
# This could just be OSH_DEBUG_STREAMS='debug crash' ? That might be
# stuffing too much into one, since a .json crash dump isn't a stream.
crash_dump_dir = posix.environ.get('OSH_CRASH_DUMP_DIR', '')
exec_deps.dumper = dev.CrashDumper(crash_dump_dir)
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
exec_deps.trace_f = trace_f
comp_lookup = completion.Lookup()
# Various Global State objects to work around readline interfaces
compopt_state = completion.OptionState()
comp_ui_state = comp_ui.State()
prompt_state = comp_ui.PromptState()
dir_stack = state.DirStack()
new_var = builtin_assign.NewVar(mem, procs, errfmt)
builtins = { # Lookup
builtin_e.ECHO: builtin_pure.Echo(exec_opts),
builtin_e.PRINTF: builtin_printf.Printf(mem, parse_ctx, errfmt),
builtin_e.PUSHD: builtin.Pushd(mem, dir_stack, errfmt),
builtin_e.POPD: builtin.Popd(mem, dir_stack, errfmt),
builtin_e.DIRS: builtin.Dirs(mem, dir_stack, errfmt),
builtin_e.PWD: builtin.Pwd(mem, errfmt),
builtin_e.READ: builtin.Read(splitter, mem),
builtin_e.HELP: builtin.Help(loader, errfmt),
builtin_e.HISTORY: builtin.History(line_input),
# Completion (more added below)
builtin_e.COMPOPT: builtin_comp.CompOpt(compopt_state, errfmt),
builtin_e.COMPADJUST: builtin_comp.CompAdjust(mem),
# test / [ differ by need_right_bracket
builtin_e.TEST: builtin_bracket.Test(False, errfmt),
builtin_e.BRACKET: builtin_bracket.Test(True, errfmt),
# Assignment (which are pure)
builtin_e.DECLARE: new_var,
builtin_e.TYPESET: new_var,
builtin_e.LOCAL: new_var,
builtin_e.EXPORT: builtin_assign.Export(mem, errfmt),
builtin_e.READONLY: builtin_assign.Readonly(mem, errfmt),
builtin_e.UNSET: builtin_assign.Unset(mem, procs, errfmt),
builtin_e.SHIFT: builtin_assign.Shift(mem),
# Pure
builtin_e.SET: builtin_pure.Set(exec_opts, mem),
builtin_e.SHOPT: builtin_pure.Shopt(exec_opts),
builtin_e.ALIAS: builtin_pure.Alias(aliases, errfmt),
builtin_e.UNALIAS: builtin_pure.UnAlias(aliases, errfmt),
builtin_e.TYPE: builtin_pure.Type(procs, aliases, exec_deps.search_path),
builtin_e.HASH: builtin_pure.Hash(exec_deps.search_path),
builtin_e.GETOPTS: builtin_pure.GetOpts(mem, errfmt),
builtin_e.COLON: lambda arg_vec: 0, # a "special" builtin
builtin_e.TRUE: lambda arg_vec: 0,
builtin_e.FALSE: lambda arg_vec: 1,
# Process
builtin_e.WAIT: builtin_process.Wait(exec_deps.waiter,
exec_deps.job_state, mem, errfmt),
builtin_e.JOBS: builtin_process.Jobs(exec_deps.job_state),
builtin_e.FG: builtin_process.Fg(exec_deps.job_state, exec_deps.waiter),
builtin_e.BG: builtin_process.Bg(exec_deps.job_state),
builtin_e.UMASK: builtin_process.Umask,
# Oil
builtin_e.REPR: builtin_oil.Repr(mem, errfmt),
builtin_e.PUSH: builtin_oil.Push(mem, errfmt),
builtin_e.USE: builtin_oil.Use(mem, errfmt),
}
ex = cmd_exec.Executor(mem, fd_state, procs, builtins, exec_opts,
parse_ctx, exec_deps)
exec_deps.ex = ex
word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, exec_deps, arena)
exec_deps.word_ev = word_ev
arith_ev = osh_expr_eval.ArithEvaluator(mem, exec_opts, word_ev, errfmt)
exec_deps.arith_ev = arith_ev
word_ev.arith_ev = arith_ev # Another circular dependency
bool_ev = osh_expr_eval.BoolEvaluator(mem, exec_opts, word_ev, errfmt)
exec_deps.bool_ev = bool_ev
expr_ev = expr_eval.OilEvaluator(mem, procs, ex, word_ev, errfmt)
exec_deps.expr_ev = expr_ev
tracer = dev.Tracer(parse_ctx, exec_opts, mem, word_ev, trace_f)
exec_deps.tracer = tracer
# HACK for circular deps
ex.word_ev = word_ev
ex.arith_ev = arith_ev
ex.bool_ev = bool_ev
ex.expr_ev = expr_ev
ex.tracer = tracer
word_ev.expr_ev = expr_ev
spec_builder = builtin_comp.SpecBuilder(ex, parse_ctx, word_ev, splitter,
comp_lookup)
# Add some builtins that depend on the executor!
complete_builtin = builtin_comp.Complete(spec_builder, comp_lookup)
builtins[builtin_e.COMPLETE] = complete_builtin
builtins[builtin_e.COMPGEN] = builtin_comp.CompGen(spec_builder)
builtins[builtin_e.CD] = builtin.Cd(mem, dir_stack, ex, errfmt)
builtins[builtin_e.JSON] = builtin_oil.Json(mem, ex, errfmt)
sig_state = process.SignalState()
sig_state.InitShell()
builtins[builtin_e.TRAP] = builtin_process.Trap(sig_state, exec_deps.traps,
exec_deps.trap_nodes, ex,
errfmt)
if lang == 'oil':
# The Oil executor wraps an OSH executor? It needs to be able to source
# it.
ex = oil_cmd_exec.OilExecutor(ex)
# PromptEvaluator rendering is needed in non-interactive shells for @P.
prompt_ev = prompt.Evaluator(lang, parse_ctx, ex, mem)
exec_deps.prompt_ev = prompt_ev
word_ev.prompt_ev = prompt_ev # HACK for circular deps
# History evaluation is a no-op if line_input is None.
hist_ev = history.Evaluator(line_input, hist_ctx, debug_f)
if opts.c is not None:
arena.PushSource(source.CFlag())
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource(source.Stdin(' -i'))
line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
line_input, prompt_state,
sig_state)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource(source.Interactive())
line_reader = reader.InteractiveLineReader(
arena, prompt_ev, hist_ev, line_input, prompt_state,
sig_state)
exec_opts.interactive = True
else:
arena.PushSource(source.Stdin(''))
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(source.MainFile(script_name))
try:
f = fd_state.Open(script_name)
except OSError as e:
ui.Stderr("osh: Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
if lang == 'osh':
c_parser = parse_ctx.MakeOshParser(line_reader)
else:
c_parser = parse_ctx.MakeOilParser(line_reader)
if exec_opts.interactive:
# Calculate ~/.config/oil/oshrc or oilrc
# Use ~/.config/oil to avoid cluttering the user's home directory. Some
# users may want to ln -s ~/.config/oil/oshrc ~/oshrc or ~/.oshrc.
# https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
home_dir = process.GetHomeDir()
assert home_dir is not None
rc_path = opts.rcfile or os_path.join(home_dir, '.config/oil',
lang + 'rc')
history_filename = os_path.join(home_dir, '.config/oil',
'history_' + lang)
if line_input:
# NOTE: We're using a different WordEvaluator here.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, exec_deps,
arena)
root_comp = completion.RootCompleter(ev, mem, comp_lookup,
compopt_state, comp_ui_state,
comp_ctx, debug_f)
term_width = 0
if opts.completion_display == 'nice':
try:
term_width = libc.get_terminal_width()
except IOError: # stdin not a terminal
pass
if term_width != 0:
display = comp_ui.NiceDisplay(term_width, comp_ui_state,
prompt_state, debug_f,
line_input)
else:
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
_InitReadline(line_input, history_filename, root_comp, display,
debug_f)
_InitDefaultCompletions(ex, complete_builtin, comp_lookup)
else: # Without readline module
display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
debug_f)
sig_state.InitInteractiveShell(display)
# NOTE: Call this AFTER _InitDefaultCompletions.
try:
SourceStartupFile(rc_path, lang, parse_ctx, ex)
except util.UserExit as e:
return e.status
line_reader.Reset() # After sourcing startup file, render $PS1
prompt_plugin = prompt.UserPlugin(mem, parse_ctx, ex)
try:
status = main_loop.Interactive(opts, ex, c_parser, display,
prompt_plugin, errfmt)
if ex.MaybeRunExitTrap():
status = ex.LastStatus()
except util.UserExit as e:
status = e.status
return status
nodes_out = [] if exec_opts.noexec else None
if nodes_out is None and opts.parser_mem_dump:
raise args.UsageError('--parser-mem-dump can only be used with -n')
_tlog('Execute(node)')
try:
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
if ex.MaybeRunExitTrap():
status = ex.LastStatus()
except util.UserExit as e:
status = e.status
# Only print nodes if the whole parse succeeded.
if nodes_out is not None and status == 0:
if opts.parser_mem_dump: # only valid in -n mode
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status