def _Eval(self, argv): # NOTE: in oil, eval shouldn't take multiple args. For clarity, 'eval ls # foo' will be an "extra arg" error. code_str = ' '.join(argv) line_reader = reader.StringLineReader(code_str, self.arena) _, c_parser = parse_lib.MakeParser(line_reader, self.arena) return self._EvalHelper(c_parser, '<eval string>')
def InitLexer(s, arena):
"""For tests only."""
match_func = match.MATCHER
line_lexer = lexer.LineLexer(match_func, '', arena)
line_reader = reader.StringLineReader(s, arena)
lx = lexer.Lexer(line_lexer, line_reader)
return line_reader, lx
def testStringLineReader(self):
arena = test_lib.MakeArena('<reader_test.py>')
r = reader.StringLineReader('one\ntwo', arena)
self.assertEqual((0, 'one\n', 0), r.GetLine())
self.assertEqual((1, 'two', 0), r.GetLine())
self.assertEqual((-1, None, 0), r.GetLine())
def testGetLine(self):
r = reader.StringLineReader('foo\nbar') # no trailing newline
self.assertEqual((-1, 'foo\n'), r.GetLine())
self.assertEqual((-1, 'bar\n'), r.GetLine())
# Keep returning EOF after exhausted
self.assertEqual((-1, None), r.GetLine())
self.assertEqual((-1, None), r.GetLine())
def MakeWordParserForPlugin(self, code_str, arena):
"""FOr $PS1, etc.
NOTE: Uses its own arena! I think that does nothing though?
"""
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
lx = lexer.Lexer(line_lexer, line_reader)
return word_parse.WordParser(self, lx, line_reader)
def MakeParserForCompletion(code_str, arena):
"""Parser for partial lines."""
# NOTE: We don't need to use a arena here? Or we need a "scratch arena" that
# doesn't interfere with the rest of the program.
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena) # AtEnd() is true
lx = lexer.Lexer(line_lexer, line_reader)
w_parser = word_parse.WordParser(lx, line_reader)
c_parser = cmd_parse.CommandParser(w_parser, lx, line_reader, arena)
return w_parser, c_parser
def _Eval(self, argv, eval_spid):
# TODO:
# - (argv, spid) should be a pattern for all builtins? They all will need
# to report usage errors.
# - set -o sane-eval should change eval to take a single string.
code_str = ' '.join(argv)
line_reader = reader.StringLineReader(code_str, self.arena)
_, c_parser = self.parse_ctx.MakeParser(line_reader)
span = self.arena.GetLineSpan(eval_spid)
path, line_num = self.arena.GetDebugInfo(span.line_id)
source_name = '<eval string from %s:%d>' % (path, line_num)
return self._EvalHelper(c_parser, source_name)
def MakeArithParser(self, code_str, arena):
"""
NOTE: We want to add tokens to a different arena, so we don't mess up the
translation.
"""
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(match.MATCHER, '', arena)
lx = lexer.Lexer(line_lexer, line_reader)
w_parser = word_parse.WordParser(self,
lx,
line_reader,
lex_mode=lex_mode_e.ARITH)
a_parser = tdop.TdopParser(arith_parse.SPEC, w_parser)
return a_parser
def MakeParserForCompletion(self, code_str, arena):
"""Parser for partial lines.
NOTE: Uses its own arena!
"""
# NOTE: We don't need to use a arena here? Or we need a "scratch arena"
# that doesn't interfere with the rest of the program.
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(match.MATCHER, '',
arena) # AtEnd() is true
lx = lexer.Lexer(line_lexer, line_reader)
w_parser = word_parse.WordParser(self, lx, line_reader)
c_parser = cmd_parse.CommandParser(self,
w_parser,
lx,
line_reader,
arena=arena)
return w_parser, c_parser
def testLineReadersAreEquivalent(self):
a1 = self.pool.NewArena()
r1 = reader.StringLineReader('one\ntwo', arena=a1)
a2 = self.pool.NewArena()
f = cStringIO.StringIO('one\ntwo')
r2 = reader.FileLineReader(f, arena=a2)
a3 = self.pool.NewArena()
lines = [(0, 'one\n'), (1, 'two\n')]
r3 = reader.VirtualLineReader(lines, a3)
for a in [a1, a2, a3]:
a.PushSource('reader_test.py')
for r in [r1, r2, r3]:
print(r)
# Lines are added to the arena with a line_id.
self.assertEqual((0, 'one\n'), r.GetLine())
self.assertEqual((1, 'two\n'), r.GetLine())
self.assertEqual((-1, None), r.GetLine())
def ParseTrapCode(self, code_str):
"""
Returns:
A node, or None if the code is invalid.
"""
line_reader = reader.StringLineReader(code_str, self.arena)
_, c_parser = parse_lib.MakeParser(line_reader, self.arena)
source_name = '<trap string>'
self.arena.PushSource(source_name)
try:
node = c_parser.ParseWholeFile()
if not node:
util.error('Parse error in %r:', source_name)
err = c_parser.Error()
ui.PrintErrorStack(err, self.arena, sys.stderr)
return None
finally:
self.arena.PopSource()
return node
def ParseTrapCode(self, code_str):
"""
Returns:
A node, or None if the code is invalid.
"""
line_reader = reader.StringLineReader(code_str, self.arena)
_, c_parser = self.parse_ctx.MakeParser(line_reader)
source_name = '<trap string>'
self.arena.PushSource(source_name)
try:
try:
node = main_loop.ParseWholeFile(c_parser)
except util.ParseError as e:
util.error('Parse error in %r:', source_name)
ui.PrettyPrintError(e, self.arena, sys.stderr)
return None
finally:
self.arena.PopSource()
return node
def OshMain(argv):
(opts, argv) = Options().parse_args(argv)
state = util.TraceState()
if 'cp' in opts.trace:
util.WrapMethods(cmd_parse.CommandParser, state)
if 'wp' in opts.trace:
util.WrapMethods(word_parse.WordParser, state)
if 'lexer' in opts.trace:
util.WrapMethods(lexer.Lexer, state)
if len(argv) == 0:
dollar0 = sys.argv[0] # e.g. bin/osh
else:
dollar0 = argv[0] # the script name
# TODO: Create a --parse action or 'osh parse' or 'oil osh-parse'
# osh-fix
# It uses a different memory-management model. It's a batch program and not
# an interactive program.
pool = Pool()
arena = pool.NewArena()
# TODO: Maybe wrap this initialization sequence up in an oil_State, like
# lua_State.
status_lines = ui.MakeStatusLines()
mem = cmd_exec.Mem(dollar0, argv[1:])
builtins = builtin.Builtins(status_lines[0])
funcs = {}
# Passed to Executor for 'complete', and passed to completion.Init
comp_lookup = completion.CompletionLookup()
exec_opts = cmd_exec.ExecOpts()
# TODO: How to get a handle to initialized builtins here?
# tokens.py has it. I think you just make a separate table, with
# metaprogramming.
ex = cmd_exec.Executor(mem, builtins, funcs, comp_lookup, exec_opts,
parse_lib.MakeParserForExecutor)
# NOTE: The rc file can contain both commands and functions... ideally we
# would only want to save nodes/lines for the functions.
try:
rc_path = 'oilrc'
with open(rc_path) as f:
contents = f.read()
arena.AddSourcePath(rc_path)
#print(repr(contents))
rc_line_reader = reader.StringLineReader(contents, arena=arena)
_, rc_c_parser = parse_lib.MakeParserForTop(rc_line_reader)
rc_node = rc_c_parser.ParseWholeFile()
if not rc_node:
# TODO: Error should return a token, and then the token should have a
# arena index, and then look that up in the arena.
err = rc_c_parser.Error()
ui.PrintError(err, arena, sys.stderr)
return 2 # parse error is code 2
status = ex.Execute(rc_node)
#print('oilrc:', status, cflow, file=sys.stderr)
# Ignore bad status?
except IOError as e:
if e.errno != errno.ENOENT:
raise
if opts.command is not None:
arena.AddSourcePath('<-c arg>')
line_reader = reader.StringLineReader(opts.command, arena=arena)
interactive = False
elif opts.interactive: # force interactive
arena.AddSourcePath('<stdin -i>')
line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena)
interactive = True
else:
try:
script_name = argv[0]
except IndexError:
if sys.stdin.isatty():
arena.AddSourcePath('<interactive>')
line_reader = reader.InteractiveLineReader(OSH_PS1,
arena=arena)
interactive = True
else:
arena.AddSourcePath('<stdin>')
line_reader = reader.StringLineReader(sys.stdin.read(),
arena=arena)
interactive = False
else:
arena.AddSourcePath(script_name)
with open(script_name) as f:
line_reader = reader.StringLineReader(f.read(), arena=arena)
interactive = False
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
w_parser, c_parser = parse_lib.MakeParserForTop(line_reader, arena=arena)
if interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts)
completion.Init(builtins, mem, funcs, comp_lookup, status_lines, ev)
# TODO: Could instantiate "printer" instead of showing ops
InteractiveLoop(opts, ex, c_parser, w_parser, line_reader)
status = 0 # TODO: set code
else:
# Parse the whole thing up front
#print('Parsing file')
# TODO: Do I need ParseAndEvalLoop? How is it different than
# InteractiveLoop?
node = c_parser.ParseWholeFile()
if not node:
err = c_parser.Error()
ui.PrintError(err, arena, sys.stderr)
return 2 # parse error is code 2
if opts.ast_output:
if opts.ast_format == 'oheap':
if opts.ast_output == '-':
if sys.stdout.isatty():
raise RuntimeError(
'ERROR: Not dumping binary data to a TTY.')
f = sys.stdout
else:
f = open(opts.ast_output, 'wb') # implicitly closed
enc = encode.Params()
out = encode.BinOutput(f)
encode.EncodeRoot(node, enc, out)
else: # text output
if opts.ast_output == '-':
f = sys.stdout
else:
f = open(opts.ast_output, 'w') # implicitly closed
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
abbrev_hook = (ast.AbbreviateNodes
if 'abbrev-' in opts.ast_format else None)
tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
if opts.do_exec:
status = ex.Execute(node)
else:
util.log('Execution skipped because --no-exec was passed')
status = 0
if opts.fix:
fix.PrintAsOil(arena, node, opts.debug_spans)
return status
def OshMain(argv, login_shell):
spec = args.FlagsAndOptions()
spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string
spec.ShortFlag('-i') # interactive
# TODO: -h too
spec.LongFlag('--help')
spec.LongFlag('--version')
spec.LongFlag('--ast-format',
['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap'],
default='abbrev-text')
spec.LongFlag('--show-ast') # execute and show
spec.LongFlag('--fix')
spec.LongFlag('--debug-spans')
spec.LongFlag('--print-status')
spec.LongFlag(
'--trace',
['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now
spec.LongFlag('--hijack-shebang')
builtin.AddOptionsToArgSpec(spec)
try:
opts, opt_index = spec.Parse(argv)
except args.UsageError as e:
util.usage(str(e))
return 2
if opts.help:
loader = util.GetResourceLoader() # TOOD: Use Global
builtin.Help(['osh-usage'], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
trace_state = util.TraceState()
if 'cmd-parse' == opts.trace:
util.WrapMethods(cmd_parse.CommandParser, trace_state)
if 'word-parse' == opts.trace:
util.WrapMethods(word_parse.WordParser, trace_state)
if 'lexer' == opts.trace:
util.WrapMethods(lexer.Lexer, trace_state)
if opt_index == len(argv):
dollar0 = sys.argv[0] # e.g. bin/osh
else:
dollar0 = argv[opt_index] # the script name, or the arg after -c
# TODO: Create a --parse action or 'osh parse' or 'oil osh-parse'
# osh-fix
# It uses a different memory-management model. It's a batch program and not
# an interactive program.
pool = alloc.Pool()
arena = pool.NewArena()
# TODO: Maybe wrap this initialization sequence up in an oil_State, like
# lua_State.
status_lines = ui.MakeStatusLines()
mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ)
funcs = {}
# Passed to Executor for 'complete', and passed to completion.Init
if completion:
comp_lookup = completion.CompletionLookup()
else:
# TODO: NullLookup?
comp_lookup = None
exec_opts = state.ExecOpts()
builtin.SetExecOpts(exec_opts, opts.opt_changes)
# TODO: How to get a handle to initialized builtins here?
# tokens.py has it. I think you just make a separate table, with
# metaprogramming.
ex = cmd_exec.Executor(mem, status_lines, funcs, completion, comp_lookup,
exec_opts, arena)
# NOTE: The rc file can contain both commands and functions... ideally we
# would only want to save nodes/lines for the functions.
try:
rc_path = 'oilrc'
arena.PushSource(rc_path)
with open(rc_path) as f:
rc_line_reader = reader.FileLineReader(f, arena=arena)
_, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
try:
rc_node = rc_c_parser.ParseWholeFile()
if not rc_node:
# TODO: Error should return a token, and then the token should have a
# arena index, and then look that up in the arena.
err = rc_c_parser.Error()
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
finally:
arena.PopSource()
status = ex.Execute(rc_node)
#print('oilrc:', status, cflow, file=sys.stderr)
# Ignore bad status?
except IOError as e:
if e.errno != errno.ENOENT:
raise
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena=arena)
interactive = False
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena)
interactive = True
else:
try:
script_name = argv[opt_index]
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
line_reader = reader.InteractiveLineReader(OSH_PS1,
arena=arena)
interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena=arena)
interactive = False
else:
arena.PushSource(script_name)
# TODO: Does this open file descriptor need to be moved beyond 3..9 ?
# Yes! See dash input.c setinputfile. It calls savefd().
# TODO: It also needs to be closed later.
try:
f = open(script_name)
except IOError as e:
util.error("Couldn't open %r: %s", script_name,
os.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena=arena)
interactive = False
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
w_parser, c_parser = parse_lib.MakeParser(line_reader, arena)
if interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if HAVE_READLINE:
ev = word_eval.CompletionWordEvaluator(mem, exec_opts)
status_out = completion.StatusOutput(status_lines, exec_opts)
completion.Init(builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
status_out, ev)
# TODO: Could instantiate "printer" instead of showing ops
InteractiveLoop(opts, ex, c_parser, w_parser, line_reader)
status = 0 # TODO: set code
else:
# Parse the whole thing up front
#print('Parsing file')
tlog('ParseWholeFile')
# TODO: Do I need ParseAndEvalLoop? How is it different than
# InteractiveLoop?
try:
node = c_parser.ParseWholeFile()
except util.ParseError as e:
ui.PrettyPrintError(e, arena, sys.stderr)
print('parse error: %s' % e.UserErrorString(), file=sys.stderr)
return 2
else:
# TODO: Remove this older form of error handling.
if not node:
err = c_parser.Error()
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
do_exec = True
if opts.fix:
osh2oil.PrintAsOil(arena, node, opts.debug_spans)
do_exec = False
if exec_opts.noexec:
do_exec = False
if exec_opts.noexec or opts.show_ast: # -n shows the AST
if opts.ast_format == 'oheap':
# TODO: Make this a separate flag?
if sys.stdout.isatty():
raise RuntimeError(
'ERROR: Not dumping binary data to a TTY.')
f = sys.stdout
enc = encode.Params()
out = encode.BinOutput(f)
encode.EncodeRoot(node, enc, out)
else: # text output
f = sys.stdout
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
abbrev_hook = (ast.AbbreviateNodes
if 'abbrev-' in opts.ast_format else None)
tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
#util.log("Execution skipped because 'noexec' is on ")
status = 0
if do_exec:
tlog('Execute(node)')
status = ex.Execute(node)
else:
status = 0
return status
def OshMain(argv0, argv, login_shell):
arg_r = args.Reader(argv)
try:
opts = OSH_SPEC.Parse(arg_r)
except args.UsageError as e:
ui.usage('osh usage error: %s', e)
return 2
if opts.help:
loader = util.GetResourceLoader()
builtin.Help(['osh-usage'], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
# TODO: This should be in interactive mode only?
builtin.RegisterSigIntHandler()
if arg_r.AtEnd():
dollar0 = argv0
has_main = False
else:
dollar0 = arg_r.Peek() # the script name, or the arg after -c
has_main = True
pool = alloc.Pool()
arena = pool.NewArena()
# NOTE: has_main is only for ${BASH_SOURCE[@} and family. Could be a
# required arg.
mem = state.Mem(dollar0,
argv[arg_r.i + 1:],
posix.environ,
arena,
has_main=has_main)
funcs = {}
comp_lookup = completion.CompletionLookup()
fd_state = process.FdState()
exec_opts = state.ExecOpts(mem, readline)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
aliases = {} # feedback between runtime and parser
parse_ctx = parse_lib.ParseContext(arena, aliases)
if opts.debug_file:
debug_f = util.DebugFile(fd_state.Open(opts.debug_file, mode='w'))
else:
debug_f = util.NullDebugFile()
debug_f.log('Debug file is %s', opts.debug_file)
# Controlled by env variable, flag, or hook?
dumper = dev.CrashDumper(posix.environ.get('OSH_CRASH_DUMP_DIR', ''))
if opts.xtrace_to_debug_file:
trace_f = debug_f
else:
trace_f = util.DebugFile(sys.stderr)
devtools = dev.DevTools(dumper, debug_f, trace_f)
ex = cmd_exec.Executor(mem, fd_state, funcs, comp_lookup, exec_opts,
parse_ctx, devtools)
# NOTE: The rc file can contain both commands and functions... ideally we
# would only want to save nodes/lines for the functions.
try:
rc_path = 'oilrc'
arena.PushSource(rc_path)
with open(rc_path) as f:
rc_line_reader = reader.FileLineReader(f, arena)
_, rc_c_parser = parse_ctx.MakeParser(rc_line_reader)
try:
status = main_loop.Batch(ex, rc_c_parser, arena)
finally:
arena.PopSource()
except IOError as e:
if e.errno != errno.ENOENT:
raise
# Needed in non-interactive shells for @P
prompt = ui.Prompt(arena, parse_ctx, ex)
ui.PROMPT = prompt
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
line_reader = reader.InteractiveLineReader(arena, prompt)
exec_opts.interactive = True
else:
try:
script_name = arg_r.Peek()
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
line_reader = reader.InteractiveLineReader(arena, prompt)
exec_opts.interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(script_name)
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
posix.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
w_parser, c_parser = parse_ctx.MakeParser(line_reader)
if exec_opts.interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if HAVE_READLINE:
splitter = legacy.SplitContext(mem) # TODO: share with executor.
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter,
arena)
progress_f = ui.StatusLine()
var_action = completion.VariablesActionInternal(ex.mem)
root_comp = completion.RootCompleter(ev, comp_lookup, var_action,
parse_ctx, progress_f,
debug_f)
completion.Init(readline, root_comp, debug_f)
_InitDefaultCompletions(ex, comp_lookup)
return main_loop.Interactive(opts, ex, c_parser, arena)
# TODO: Remove this after removing it from benchmarks/osh-runtime. It's no
# longer relevant with main_loop.
if opts.parser_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
nodes_out = [] if exec_opts.noexec else None
_tlog('Execute(node)')
status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)
if nodes_out is not None:
ui.PrintAst(nodes_out, opts)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % posix.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
# NOTE: We haven't closed the file opened with fd_state.Open
return status
def testStringLineReader(self):
# No Arena, gives -1
r = reader.StringLineReader('one\ntwo')
self.assertEqual((-1, 'one\n'), r.GetLine())
self.assertEqual((-1, 'two\n'), r.GetLine())
self.assertEqual((-1, None), r.GetLine())
def MakeWordParserForPlugin(code_str, arena):
line_reader = reader.StringLineReader(code_str, arena)
line_lexer = lexer.LineLexer(_MakeMatcher(), '', arena)
lx = lexer.Lexer(line_lexer, line_reader)
return word_parse.WordParser(lx, line_reader)
def InitLexer(s, arena=None):
"""For tests only."""
line_lexer = lexer.LineLexer(lex.LEXER_DEF, '', arena=arena)
line_reader = reader.StringLineReader(s, arena=arena)
lx = lexer.Lexer(line_lexer, line_reader)
return line_reader, lx
def OshMain(argv0, argv, login_shell):
spec = args.FlagsAndOptions()
spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string
spec.ShortFlag('-i') # interactive
# TODO: -h too
spec.LongFlag('--help')
spec.LongFlag('--version')
spec.LongFlag(
'--ast-format',
['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap', 'none'],
default='abbrev-text')
spec.LongFlag('--show-ast') # execute and show
spec.LongFlag('--fix')
spec.LongFlag('--debug-spans') # For oshc translate
spec.LongFlag('--print-status')
spec.LongFlag(
'--trace',
['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now
spec.LongFlag('--hijack-shebang')
# For benchmarks/*.sh
spec.LongFlag('--parser-mem-dump', args.Str)
spec.LongFlag('--runtime-mem-dump', args.Str)
builtin.AddOptionsToArgSpec(spec)
try:
opts, opt_index = spec.Parse(argv)
except args.UsageError as e:
util.usage(str(e))
return 2
if opts.help:
loader = util.GetResourceLoader()
builtin.Help(['osh-usage'], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
trace_state = util.TraceState()
if 'cmd-parse' == opts.trace:
util.WrapMethods(cmd_parse.CommandParser, trace_state)
if 'word-parse' == opts.trace:
util.WrapMethods(word_parse.WordParser, trace_state)
if 'lexer' == opts.trace:
util.WrapMethods(lexer.Lexer, trace_state)
if opt_index == len(argv):
dollar0 = argv0
else:
dollar0 = argv[opt_index] # the script name, or the arg after -c
# TODO: Create a --parse action or 'osh parse' or 'oil osh-parse'
# osh-fix
# It uses a different memory-management model. It's a batch program and not
# an interactive program.
pool = alloc.Pool()
arena = pool.NewArena()
# TODO: Maybe wrap this initialization sequence up in an oil_State, like
# lua_State.
status_lines = ui.MakeStatusLines()
mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ, arena)
funcs = {}
# Passed to Executor for 'complete', and passed to completion.Init
if completion:
comp_lookup = completion.CompletionLookup()
else:
# TODO: NullLookup?
comp_lookup = None
exec_opts = state.ExecOpts(mem)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
fd_state = process.FdState()
ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, readline,
completion, comp_lookup, exec_opts, arena)
# NOTE: The rc file can contain both commands and functions... ideally we
# would only want to save nodes/lines for the functions.
try:
rc_path = 'oilrc'
arena.PushSource(rc_path)
with open(rc_path) as f:
rc_line_reader = reader.FileLineReader(f, arena)
_, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
try:
rc_node = rc_c_parser.ParseWholeFile()
if not rc_node:
err = rc_c_parser.Error()
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
finally:
arena.PopSource()
status = ex.Execute(rc_node)
#print('oilrc:', status, cflow, file=sys.stderr)
# Ignore bad status?
except IOError as e:
if e.errno != errno.ENOENT:
raise
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
exec_opts.interactive = True
else:
try:
script_name = argv[opt_index]
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
exec_opts.interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(script_name)
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
os.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
w_parser, c_parser = parse_lib.MakeParser(line_reader, arena)
if exec_opts.interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if HAVE_READLINE:
splitter = legacy.SplitContext(mem)
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
status_out = completion.StatusOutput(status_lines, exec_opts)
completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
status_out, ev)
return InteractiveLoop(opts, ex, c_parser, w_parser, line_reader)
else:
# Parse the whole thing up front
#print('Parsing file')
_tlog('ParseWholeFile')
# TODO: Do I need ParseAndEvalLoop? How is it different than
# InteractiveLoop?
try:
node = c_parser.ParseWholeFile()
except util.ParseError as e:
ui.PrettyPrintError(e, arena, sys.stderr)
print('parse error: %s' % e.UserErrorString(), file=sys.stderr)
return 2
else:
# TODO: Remove this older form of error handling.
if not node:
err = c_parser.Error()
assert err, err # can't be empty
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
do_exec = True
if opts.fix:
#log('SPANS: %s', arena.spans)
osh2oil.PrintAsOil(arena, node, opts.debug_spans)
do_exec = False
if exec_opts.noexec:
do_exec = False
# Do this after parsing the entire file. There could be another option to
# do it before exiting runtime?
if opts.parser_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % os.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
# -n prints AST, --show-ast prints and executes
if exec_opts.noexec or opts.show_ast:
if opts.ast_format == 'none':
print('AST not printed.', file=sys.stderr)
elif opts.ast_format == 'oheap':
# TODO: Make this a separate flag?
if sys.stdout.isatty():
raise RuntimeError(
'ERROR: Not dumping binary data to a TTY.')
f = sys.stdout
enc = encode.Params()
out = encode.BinOutput(f)
encode.EncodeRoot(node, enc, out)
else: # text output
f = sys.stdout
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
abbrev_hook = (ast_lib.AbbreviateNodes
if 'abbrev-' in opts.ast_format else None)
tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
#util.log("Execution skipped because 'noexec' is on ")
status = 0
if do_exec:
_tlog('Execute(node)')
status = ex.ExecuteAndRunExitTrap(node)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % os.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump,
'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
else:
status = 0
return status
def _Eval(self, argv): # TODO: set -o sane-eval should change eval to code_str = ' '.join(argv) line_reader = reader.StringLineReader(code_str, self.arena) _, c_parser = parse_lib.MakeParser(line_reader, self.arena) return self._EvalHelper(c_parser, '<eval string>')
