def PrintAst(node, flag):
# type: (command_t, arg_types.main) -> None
if flag.ast_format == 'none':
stderr_line('AST not printed.')
if 0:
from _devbuild.gen.id_kind_asdl import Id_str
from frontend.lexer import ID_HIST
for id_, count in ID_HIST.most_common(10):
print('%8d %s' % (count, Id_str(id_)))
print()
total = sum(ID_HIST.values())
print('%8d total tokens returned' % total)
else: # text output
f = mylib.Stdout()
afmt = flag.ast_format # note: mycpp rewrite to avoid 'in'
if afmt in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif afmt in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError()
if 'abbrev-' in afmt:
tree = node.AbbreviatedTree()
else:
tree = node.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
def t_parse(
s, # type: str
expected=None, # type: Optional[str]
):
# type: (...) -> arith_expr_t
p = typed_arith_parse.MakeParser(s)
tree = p.Parse()
print()
print('---')
print()
print(s)
print()
print(tree)
import sys
from asdl import format as fmt
ast_f = fmt.DetectConsoleOutput(sys.stdout)
p_node = tree.AbbreviatedTree()
fmt.PrintTree(p_node, ast_f)
#v = PrettyPrinter()
#v.Visit(tree)
#print('%-40s %s' % (s, sexpr))
return tree
def main(argv):
# type: (List[str]) -> int
arena = alloc.Arena()
arena.PushSource(source.Stdin(''))
parse_opts = parse_lib.OilParseOptions()
# Dummy value; not respecting aliases!
aliases = {} # type: Dict[str, Any]
# parse `` and a[x+1]=bar differently
loader = pyutil.GetResourceLoader()
oil_grammar = meta.LoadOilGrammar(loader)
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar,
one_pass_parse=True)
line_reader = reader.FileLineReader(mylib.Stdin(), arena)
c_parser = parse_ctx.MakeOshParser(line_reader)
try:
node = main_loop.ParseWholeFile(c_parser)
except util.ParseError as e:
ui.PrettyPrintError(e, arena)
return 2
assert node is not None
tree = node.AbbreviatedTree()
#tree = node.PrettyTree()
ast_f = fmt.DetectConsoleOutput(mylib.Stdout())
fmt.PrintTree(tree, ast_f)
ast_f.write('\n')
return 0
def main(argv):
# type: (List[str]) -> int
pool = alloc.Pool()
arena = pool.NewArena()
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena)
# Dummy value; not respecting aliases!
aliases = {} # type: Dict[str, Any]
# parse `` and a[x+1]=bar differently
parse_ctx = parse_lib.ParseContext(arena, aliases, one_pass_parse=True)
c_parser = parse_ctx.MakeOshParser(line_reader)
try:
node = main_loop.ParseWholeFile(c_parser)
except util.ParseError as e:
ui.PrettyPrintError(e, arena)
return 2
assert node is not None
tree = node.AbbreviatedTree()
#tree = node.PrettyTree()
ast_f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, ast_f)
ast_f.write('\n')
return 0
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py': # Prints the module
# Called by asdl/run.sh py-cpp
schema_path = argv[2]
app_types = {'id': asdl.UserType('id_kind_asdl', 'Id_t')}
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(schema_ast, root, type_lookup)
log('AST for this ASDL schema:')
schema_ast.Print(sys.stdout, 0)
print()
log('Dynamically created a Python module with these types:')
for name in dir(root):
print('\t' + name)
if 1:
# NOTE: It can be pickled, but not marshaled
import marshal
import cPickle
print(dir(marshal))
out_path = schema_path + '.pickle'
with open(out_path, 'w') as f:
#marshal.dump(type_lookup, f)
# Version 2 is the highest protocol for Python 2.7.
cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2)
print('runtime_type_lookup:')
for name, desc in type_lookup.runtime_type_lookup.items():
print(name)
print(desc)
print()
print('Wrote %s' % out_path)
elif action == 'arith-format': # pretty printing
expr = argv[2]
obj = typed_arith_parse.ParseShell(expr)
tree = obj.PrettyTree()
#treee= ['hi', 'there', ['a', 'b'], 'c']
f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, f)
print()
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def PrintAst(nodes, opts):
if len(nodes) == 1:
node = nodes[0]
else:
node = command.CommandList(nodes)
if opts.ast_format == 'none':
print('AST not printed.', file=sys.stderr)
else: # text output
f = sys.stdout
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
abbrev_hook = (ast_lib.AbbreviateNodes
if 'abbrev-' in opts.ast_format else None)
tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
def __repr__(self):
ast_f = fmt.TextOutput(util.Buffer()) # No color by default.
#ast_f = fmt.AnsiOutput(io.StringIO())
tree = fmt.MakeTree(self)
fmt.PrintTree(tree, ast_f)
s, _ = ast_f.GetRaw()
return s
def PrintAst(nodes, opts):
if len(nodes) == 1:
node = nodes[0]
else:
node = ast.CommandList(nodes)
if opts.ast_format == 'none':
print('AST not printed.', file=sys.stderr)
elif opts.ast_format == 'oheap':
# TODO: Make this a separate flag?
if sys.stdout.isatty():
raise RuntimeError('ERROR: Not dumping binary data to a TTY.')
f = sys.stdout
enc = encode.Params()
out = encode.BinOutput(f)
encode.EncodeRoot(node, enc, out)
else: # text output
f = sys.stdout
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
abbrev_hook = (ast_lib.AbbreviateNodes
if 'abbrev-' in opts.ast_format else None)
tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
def main(argv):
# type: (List[str]) -> int
arena = alloc.Arena()
arena.PushSource(source.Stdin(''))
loader = pyutil.GetResourceLoader()
oil_grammar = meta.LoadOilGrammar(loader)
parse_ctx = None
e_parser = expr_parse.ExprParser(parse_ctx, oil_grammar)
line_lexer = lexer.LineLexer('', arena)
line_reader = reader.FileLineReader(sys.stdin, arena)
lex = lexer.Lexer(line_lexer, line_reader)
try:
pnode, _ = e_parser.Parse(lex, grammar_nt.command_expr)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
return 2
#print(pnode)
tr = expr_to_ast.Transformer(oil_grammar)
node = tr.Expr(pnode)
assert node is not None
tree = node.AbbreviatedTree()
#tree = node.PrettyTree()
ast_f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, ast_f)
ast_f.write('\n')
return 0
def PrintAst(nodes, opts):
# type: (List[command_t], Any) -> None
if len(nodes) == 1:
node = nodes[0]
else:
node = command.CommandList(nodes)
if opts.ast_format == 'none':
print('AST not printed.', file=sys.stderr)
else: # text output
f = sys.stdout
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
if 'abbrev-' in opts.ast_format:
tree = node.AbbreviatedTree()
else:
tree = node.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
arg0 = cmd_val.argv[0]
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
log('Unhandled SimpleCommand')
f = mylib.Stdout()
#ast_f = fmt.DetectConsoleOutput(f)
# Stupid Eclipse debugger doesn't display ANSI
ast_f = fmt.TextOutput(f)
tree = cmd_val.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
return 0
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py': # Prints the module
schema_path = argv[2]
with open(schema_path) as f:
module = asdl.parse(f)
app_types = {'id': asdl.UserType(Id)}
type_lookup = asdl.ResolveTypes(module, app_types)
# Note this is a big tree. But we really want a graph of pointers to
# instances.
# Type(name, Product(...))
# Type(name, Sum([Constructor(...), ...]))
#print(module)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(module, root, type_lookup)
print('Dynamically created a Python module with these types:')
for name in dir(root):
print('\t' + name)
elif action == 'arith-encode': # oheap encoding
expr = argv[2]
out_path = argv[3]
obj = arith_parse.ParseShell(expr)
print('Encoding %r into binary:' % expr)
print(obj)
enc = encode.Params()
with open(out_path, 'wb') as f:
out = encode.BinOutput(f)
encode.EncodeRoot(obj, enc, out)
elif action == 'arith-format': # pretty printing
expr = argv[2]
obj = arith_parse.ParseShell(expr)
#out = fmt.TextOutput(sys.stdout)
tree = fmt.MakeTree(obj)
#treee= ['hi', 'there', ['a', 'b'], 'c']
f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, f)
print()
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(
cmd_val.arg_spids) else runtime.NO_SPID
arg0 = argv[0]
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# Copied from core/executor.py
if call_procs:
proc_node = self.procs.get(arg0)
if proc_node is not None:
if (self.exec_opts.strict_errexit()
and self.mutable_opts.ErrExitIsDisabled()):
# TODO: make errfmt a member
#self.errfmt.Print_('errexit was disabled for this construct',
# span_id=self.mutable_opts.errexit.spid_stack[0])
#stderr_line('')
e_die(
"Can't run a proc while errexit is disabled. "
"Use 'catch' or wrap it in a process with $0 myproc",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(proc_node, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# See how many tests will pass
#if mylib.PYTHON:
if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling
import subprocess
try:
status = subprocess.call(cmd_val.argv)
except OSError as e:
log('Error running %s: %s', cmd_val.argv, e)
return 1
return status
log('Unhandled SimpleCommand')
f = mylib.Stdout()
#ast_f = fmt.DetectConsoleOutput(f)
# Stupid Eclipse debugger doesn't display ANSI
ast_f = fmt.TextOutput(f)
tree = cmd_val.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
return 0
def __repr__(self): # TODO: Break this circular dependency. from asdl import format as fmt ast_f = fmt.TextOutput(util.Buffer()) # No color by default. tree = fmt.MakeTree(self) fmt.PrintTree(tree, ast_f) s, _ = ast_f.GetRaw() return s
def PrettyPrint(self, f=sys.stdout):
# type: (IO[str]) -> None
"""Print abbreviated tree in color, for debugging."""
from asdl import format as fmt
ast_f = fmt.DetectConsoleOutput(f)
tree = self.AbbreviatedTree()
fmt.PrintTree(tree, ast_f)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(
cmd_val.arg_spids) else runtime.NO_SPID
arg0 = argv[0]
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
func_node = self.procs.get(arg0)
if func_node is not None:
if (self.exec_opts.strict_errexit()
and self.mutable_opts.errexit.SpidIfDisabled() !=
runtime.NO_SPID):
# NOTE: This would be checked below, but this gives a better error
# message.
e_die(
"can't disable errexit running a function. "
"Maybe wrap the function in a process with the at-splice "
"pattern.",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(func_node, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# See how many tests will pass
#if mylib.PYTHON:
if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling
import subprocess
try:
status = subprocess.call(cmd_val.argv)
except OSError as e:
log('Error running %s: %s', cmd_val.argv, e)
return 1
return status
log('Unhandled SimpleCommand')
f = mylib.Stdout()
#ast_f = fmt.DetectConsoleOutput(f)
# Stupid Eclipse debugger doesn't display ANSI
ast_f = fmt.TextOutput(f)
tree = cmd_val.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
return 0
def PrettyPrint(self, f=None):
# type: (Optional[mylib.Writer]) -> None
"""Print abbreviated tree in color, for debugging."""
from asdl import format as fmt
f = f if f else mylib.Stdout()
ast_f = fmt.DetectConsoleOutput(f)
tree = self.AbbreviatedTree()
fmt.PrintTree(tree, ast_f)
def __repr__(self):
# type: () -> str
# TODO: Break this circular dependency.
from asdl import format as fmt
ast_f = fmt.TextOutput(StringIO()) # No color by default.
tree = self.PrettyTree()
fmt.PrintTree(tree, ast_f)
s, _ = ast_f.GetRaw()
return s
def testRepeatedString(self):
node = demo_asdl.assign('declare', ['-r', '-x'])
f = cStringIO.StringIO()
f1 = fmt.TextOutput(f)
f2 = fmt.HtmlOutput(f)
for ast_f in [f1, f2]:
tree = node.PrettyTree()
fmt.PrintTree(tree, ast_f)
pretty_str = f.getvalue()
print(pretty_str)
if ast_f is f1:
self.assertEqual('(assign name:declare flags:[-r -x])', pretty_str)
t2 = node.AbbreviatedTree()
fmt.PrintTree(t2, ast_f)
def testRepeatedString(self):
node = arith_ast.assign('declare', ['-r', '-x'])
f = cStringIO.StringIO()
ast_f = fmt.TextOutput(f)
tree = fmt.MakeTree(node)
#print(tree)
fmt.PrintTree(tree, ast_f)
pretty_str = f.getvalue()
print(pretty_str)
self.assertEqual('(assign name:declare flags:[-r -x])', pretty_str)
def testSimpleSum(self):
node = arith_ast.op_id_e.Plus
print(node)
f = cStringIO.StringIO()
ast_f = fmt.TextOutput(f)
tree = fmt.MakeTree(node)
fmt.PrintTree(tree, ast_f)
# Hm this prints 'Plus'. Doesn't print the class or the number.
# But those aren't intrinsic. These are mostly used for ther IDENTITY.
# I think the ASDL_TYPE field contains the relevant info. Yes!
pretty_str = f.getvalue()
print(pretty_str)
def run_tests():
# type: () -> None
lex = Lexer('abc')
while True:
tok_type, tok_val = lex.Read()
if tok_type == tok_e.Eof:
break
#print('%s %s' % (tok_type, tok_val))
log('tok_val %s', tok_val)
CASES = [
'1+2',
'1+2*3',
'1*2+3',
'(1+2)*3',
'a+b+c+d',
'a*b*3*4',
# expect errors here:
'(',
')',
'(a+b',
' ',
' $$ ',
]
for expr in CASES:
lex = Lexer(expr)
p = Parser(lex)
log('')
log('--')
log('%s =>', expr)
tree = None # type: Optional[expr_t]
try:
tree = p.Parse()
except ParseError as e:
log('Parse error: %s', e.msg)
continue
#log('%s', tree)
htree = tree.AbbreviatedTree()
ast_f = fmt.AnsiOutput(mylib.Stdout())
fmt.PrintTree(htree, ast_f)
ast_f.write('\n')
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py':
schema_path = argv[2]
module = asdl.parse(schema_path)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(module, root, app_types={'id': asdl.UserType(Id)})
print(dir(root))
elif action == 'arith-encode':
expr = argv[2]
out_path = argv[3]
obj = arith_parse.ParseShell(expr)
print('Encoding %r into binary:' % expr)
print(obj)
enc = encode.Params()
with open(out_path, 'wb') as f:
out = encode.BinOutput(f)
encode.EncodeRoot(obj, enc, out)
elif action == 'arith-format':
expr = argv[2]
obj = arith_parse.ParseShell(expr)
#out = fmt.TextOutput(sys.stdout)
tree = fmt.MakeTree(obj)
#treee= ['hi', 'there', ['a', 'b'], 'c']
fmt.PrintTree(tree, sys.stdout)
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def PrintAst(nodes, opts):
# type: (List[command_t], Any) -> None
if len(nodes) == 1:
node = nodes[0]
else:
node = command.CommandList(nodes)
if opts.ast_format == 'none':
print('AST not printed.', file=sys.stderr)
if 0:
from _devbuild.gen.id_kind_asdl import Id_str
from frontend.lexer import ID_HIST
for id_, count in ID_HIST.most_common(10):
print('%8d %s' % (count, Id_str(id_)))
print()
total = sum(ID_HIST.values())
print('%8d total tokens returned' % total)
else: # text output
f = mylib.Stdout()
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError()
if 'abbrev-' in opts.ast_format:
tree = node.AbbreviatedTree()
else:
tree = node.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
def OshMain(argv0, argv, login_shell):
spec = args.FlagsAndOptions()
spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string
spec.ShortFlag('-i') # interactive
# TODO: -h too
spec.LongFlag('--help')
spec.LongFlag('--version')
spec.LongFlag(
'--ast-format',
['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap', 'none'],
default='abbrev-text')
spec.LongFlag('--show-ast') # execute and show
spec.LongFlag('--fix')
spec.LongFlag('--debug-spans') # For oshc translate
spec.LongFlag('--print-status')
spec.LongFlag(
'--trace',
['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now
spec.LongFlag('--hijack-shebang')
# For benchmarks/*.sh
spec.LongFlag('--parser-mem-dump', args.Str)
spec.LongFlag('--runtime-mem-dump', args.Str)
builtin.AddOptionsToArgSpec(spec)
try:
opts, opt_index = spec.Parse(argv)
except args.UsageError as e:
util.usage(str(e))
return 2
if opts.help:
loader = util.GetResourceLoader()
builtin.Help(['osh-usage'], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
trace_state = util.TraceState()
if 'cmd-parse' == opts.trace:
util.WrapMethods(cmd_parse.CommandParser, trace_state)
if 'word-parse' == opts.trace:
util.WrapMethods(word_parse.WordParser, trace_state)
if 'lexer' == opts.trace:
util.WrapMethods(lexer.Lexer, trace_state)
if opt_index == len(argv):
dollar0 = argv0
else:
dollar0 = argv[opt_index] # the script name, or the arg after -c
# TODO: Create a --parse action or 'osh parse' or 'oil osh-parse'
# osh-fix
# It uses a different memory-management model. It's a batch program and not
# an interactive program.
pool = alloc.Pool()
arena = pool.NewArena()
# TODO: Maybe wrap this initialization sequence up in an oil_State, like
# lua_State.
status_lines = ui.MakeStatusLines()
mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ, arena)
funcs = {}
# Passed to Executor for 'complete', and passed to completion.Init
if completion:
comp_lookup = completion.CompletionLookup()
else:
# TODO: NullLookup?
comp_lookup = None
exec_opts = state.ExecOpts(mem)
builtin.SetExecOpts(exec_opts, opts.opt_changes)
fd_state = process.FdState()
ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, readline,
completion, comp_lookup, exec_opts, arena)
# NOTE: The rc file can contain both commands and functions... ideally we
# would only want to save nodes/lines for the functions.
try:
rc_path = 'oilrc'
arena.PushSource(rc_path)
with open(rc_path) as f:
rc_line_reader = reader.FileLineReader(f, arena)
_, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
try:
rc_node = rc_c_parser.ParseWholeFile()
if not rc_node:
err = rc_c_parser.Error()
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
finally:
arena.PopSource()
status = ex.Execute(rc_node)
#print('oilrc:', status, cflow, file=sys.stderr)
# Ignore bad status?
except IOError as e:
if e.errno != errno.ENOENT:
raise
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena)
if opts.i: # -c and -i can be combined
exec_opts.interactive = True
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
exec_opts.interactive = True
else:
try:
script_name = argv[opt_index]
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
line_reader = reader.InteractiveLineReader(OSH_PS1, arena)
exec_opts.interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena)
else:
arena.PushSource(script_name)
try:
f = fd_state.Open(script_name)
except OSError as e:
util.error("Couldn't open %r: %s", script_name,
os.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena)
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
w_parser, c_parser = parse_lib.MakeParser(line_reader, arena)
if exec_opts.interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if HAVE_READLINE:
splitter = legacy.SplitContext(mem)
ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
status_out = completion.StatusOutput(status_lines, exec_opts)
completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
status_out, ev)
return InteractiveLoop(opts, ex, c_parser, w_parser, line_reader)
else:
# Parse the whole thing up front
#print('Parsing file')
_tlog('ParseWholeFile')
# TODO: Do I need ParseAndEvalLoop? How is it different than
# InteractiveLoop?
try:
node = c_parser.ParseWholeFile()
except util.ParseError as e:
ui.PrettyPrintError(e, arena, sys.stderr)
print('parse error: %s' % e.UserErrorString(), file=sys.stderr)
return 2
else:
# TODO: Remove this older form of error handling.
if not node:
err = c_parser.Error()
assert err, err # can't be empty
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
do_exec = True
if opts.fix:
#log('SPANS: %s', arena.spans)
osh2oil.PrintAsOil(arena, node, opts.debug_spans)
do_exec = False
if exec_opts.noexec:
do_exec = False
# Do this after parsing the entire file. There could be another option to
# do it before exiting runtime?
if opts.parser_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % os.getpid()
with open(input_path) as f, open(opts.parser_mem_dump, 'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--parser-mem-dump)', input_path,
opts.parser_mem_dump)
# -n prints AST, --show-ast prints and executes
if exec_opts.noexec or opts.show_ast:
if opts.ast_format == 'none':
print('AST not printed.', file=sys.stderr)
elif opts.ast_format == 'oheap':
# TODO: Make this a separate flag?
if sys.stdout.isatty():
raise RuntimeError(
'ERROR: Not dumping binary data to a TTY.')
f = sys.stdout
enc = encode.Params()
out = encode.BinOutput(f)
encode.EncodeRoot(node, enc, out)
else: # text output
f = sys.stdout
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
abbrev_hook = (ast_lib.AbbreviateNodes
if 'abbrev-' in opts.ast_format else None)
tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
#util.log("Execution skipped because 'noexec' is on ")
status = 0
if do_exec:
_tlog('Execute(node)')
status = ex.ExecuteAndRunExitTrap(node)
# NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
# don't because they call sys.exit().
if opts.runtime_mem_dump:
# This might be superstition, but we want to let the value stabilize
# after parsing. bash -c 'cat /proc/$$/status' gives different results
# with a sleep.
time.sleep(0.001)
input_path = '/proc/%d/status' % os.getpid()
with open(input_path) as f, open(opts.runtime_mem_dump,
'w') as f2:
contents = f.read()
f2.write(contents)
log('Wrote %s to %s (--runtime-mem-dump)', input_path,
opts.runtime_mem_dump)
else:
status = 0
return status
def _PrettyPrint(n):
"""Prints in color."""
ast_f = fmt.DetectConsoleOutput(sys.stdout)
tree = n.PrettyTree()
fmt.PrintTree(tree, ast_f)
def main(argv):
# type: (List[str]) -> int
arena = alloc.Arena()
opt_array = [False] * option_i.ARRAY_SIZE
parse_opts = optview.Parse(opt_array)
# Dummy value; not respecting aliases!
aliases = {} # type: Dict[str, str]
# parse `` and a[x+1]=bar differently
oil_grammar = None # type: Grammar
if mylib.PYTHON:
loader = pyutil.GetResourceLoader()
oil_grammar = pyutil.LoadOilGrammar(loader)
parse_ctx = parse_lib.ParseContext(arena, parse_opts, aliases, oil_grammar)
pretty_print = True
if len(argv) == 1:
line_reader = reader.FileLineReader(mylib.Stdin(), arena)
src = source.Stdin('') # type: source_t
elif len(argv) == 2:
path = argv[1]
f = mylib.open(path)
line_reader = reader.FileLineReader(f, arena)
src = source.MainFile(path)
elif len(argv) == 3:
if argv[1] == '-c':
# This path is easier to run through GDB
line_reader = reader.StringLineReader(argv[2], arena)
src = source.CFlag()
elif argv[1] == '-n': # For benchmarking, allow osh_parse -n file.txt
path = argv[2]
f = mylib.open(path)
line_reader = reader.FileLineReader(f, arena)
src = source.MainFile(path)
# This is like --ast-format none, which benchmarks/osh-helper.sh passes.
pretty_print = False
else:
raise AssertionError()
else:
raise AssertionError()
arena.PushSource(src)
c_parser = parse_ctx.MakeOshParser(line_reader)
try:
#node = main_loop.ParseWholeFile(c_parser)
node = ParseWholeFile(c_parser)
except error.Parse as e:
ui.PrettyPrintError(e, arena)
return 2
assert node is not None
# C++ doesn't have the abbreviations yet (though there are some differences
# like omitting spids)
#tree = node.AbbreviatedTree()
if pretty_print:
tree = node.PrettyTree()
ast_f = fmt.DetectConsoleOutput(mylib.Stdout())
fmt.PrintTree(tree, ast_f)
ast_f.write('\n')
return 0
def _PrettyPrint(n): from asdl import format as fmt ast_f = fmt.DetectConsoleOutput(sys.stdout) tree = fmt.MakeTree(n) fmt.PrintTree(tree, ast_f)
def run_tests():
# type: () -> None
lex = Lexer('abc')
while True:
tok_type, tok_val = lex.Read()
if tok_type == tok_e.Eof:
break
#print('%s %s' % (tok_type, tok_val))
log('tok_val %s', tok_val)
CASES = [
'1+2',
'1+2*3',
'1*2+3',
'(1+2)*3',
'a+b+c+d',
'a*b*3*4',
'1',
'a',
# expect errors here:
'(',
')',
'(a+b',
' ',
' $$ ',
]
for expr in CASES:
lex = Lexer(expr)
p = Parser(lex)
log('')
log('--')
log('%s =>', expr)
node = None # type: Optional[expr_t]
try:
node = p.Parse()
except ParseError as e:
log('Parse error: %s', e.msg)
continue
#log('%s', tree)
htree = node.AbbreviatedTree()
ast_f = fmt.AnsiOutput(mylib.Stdout())
fmt.PrintTree(htree, ast_f)
ast_f.write('\n')
UP_node = node
with tagswitch(UP_node) as case:
if case(expr_e.Const):
node = cast(expr__Const, UP_node)
log('Const %d', node.i)
elif case(expr_e.Var):
node = cast(expr__Var, UP_node)
log('Var %s', node.name)
else:
log('Other')
def OshMain(argv, login_shell):
spec = args.FlagsAndOptions()
spec.ShortFlag('-c', args.Str, quit_parsing_flags=True) # command string
spec.ShortFlag('-i') # interactive
# TODO: -h too
spec.LongFlag('--help')
spec.LongFlag('--version')
spec.LongFlag('--ast-format',
['text', 'abbrev-text', 'html', 'abbrev-html', 'oheap'],
default='abbrev-text')
spec.LongFlag('--show-ast') # execute and show
spec.LongFlag('--fix')
spec.LongFlag('--debug-spans')
spec.LongFlag('--print-status')
spec.LongFlag(
'--trace',
['cmd-parse', 'word-parse', 'lexer']) # NOTE: can only trace one now
spec.LongFlag('--hijack-shebang')
builtin.AddOptionsToArgSpec(spec)
try:
opts, opt_index = spec.Parse(argv)
except args.UsageError as e:
util.usage(str(e))
return 2
if opts.help:
loader = util.GetResourceLoader() # TOOD: Use Global
builtin.Help(['osh-usage'], loader)
return 0
if opts.version:
# OSH version is the only binary in Oil right now, so it's all one version.
_ShowVersion()
return 0
trace_state = util.TraceState()
if 'cmd-parse' == opts.trace:
util.WrapMethods(cmd_parse.CommandParser, trace_state)
if 'word-parse' == opts.trace:
util.WrapMethods(word_parse.WordParser, trace_state)
if 'lexer' == opts.trace:
util.WrapMethods(lexer.Lexer, trace_state)
if opt_index == len(argv):
dollar0 = sys.argv[0] # e.g. bin/osh
else:
dollar0 = argv[opt_index] # the script name, or the arg after -c
# TODO: Create a --parse action or 'osh parse' or 'oil osh-parse'
# osh-fix
# It uses a different memory-management model. It's a batch program and not
# an interactive program.
pool = alloc.Pool()
arena = pool.NewArena()
# TODO: Maybe wrap this initialization sequence up in an oil_State, like
# lua_State.
status_lines = ui.MakeStatusLines()
mem = state.Mem(dollar0, argv[opt_index + 1:], os.environ)
funcs = {}
# Passed to Executor for 'complete', and passed to completion.Init
if completion:
comp_lookup = completion.CompletionLookup()
else:
# TODO: NullLookup?
comp_lookup = None
exec_opts = state.ExecOpts()
builtin.SetExecOpts(exec_opts, opts.opt_changes)
# TODO: How to get a handle to initialized builtins here?
# tokens.py has it. I think you just make a separate table, with
# metaprogramming.
ex = cmd_exec.Executor(mem, status_lines, funcs, completion, comp_lookup,
exec_opts, arena)
# NOTE: The rc file can contain both commands and functions... ideally we
# would only want to save nodes/lines for the functions.
try:
rc_path = 'oilrc'
arena.PushSource(rc_path)
with open(rc_path) as f:
rc_line_reader = reader.FileLineReader(f, arena=arena)
_, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
try:
rc_node = rc_c_parser.ParseWholeFile()
if not rc_node:
# TODO: Error should return a token, and then the token should have a
# arena index, and then look that up in the arena.
err = rc_c_parser.Error()
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
finally:
arena.PopSource()
status = ex.Execute(rc_node)
#print('oilrc:', status, cflow, file=sys.stderr)
# Ignore bad status?
except IOError as e:
if e.errno != errno.ENOENT:
raise
if opts.c is not None:
arena.PushSource('<command string>')
line_reader = reader.StringLineReader(opts.c, arena=arena)
interactive = False
elif opts.i: # force interactive
arena.PushSource('<stdin -i>')
line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena)
interactive = True
else:
try:
script_name = argv[opt_index]
except IndexError:
if sys.stdin.isatty():
arena.PushSource('<interactive>')
line_reader = reader.InteractiveLineReader(OSH_PS1,
arena=arena)
interactive = True
else:
arena.PushSource('<stdin>')
line_reader = reader.FileLineReader(sys.stdin, arena=arena)
interactive = False
else:
arena.PushSource(script_name)
# TODO: Does this open file descriptor need to be moved beyond 3..9 ?
# Yes! See dash input.c setinputfile. It calls savefd().
# TODO: It also needs to be closed later.
try:
f = open(script_name)
except IOError as e:
util.error("Couldn't open %r: %s", script_name,
os.strerror(e.errno))
return 1
line_reader = reader.FileLineReader(f, arena=arena)
interactive = False
# TODO: assert arena.NumSourcePaths() == 1
# TODO: .rc file needs its own arena.
w_parser, c_parser = parse_lib.MakeParser(line_reader, arena)
if interactive:
# NOTE: We're using a different evaluator here. The completion system can
# also run functions... it gets the Executor through Executor._Complete.
if HAVE_READLINE:
ev = word_eval.CompletionWordEvaluator(mem, exec_opts)
status_out = completion.StatusOutput(status_lines, exec_opts)
completion.Init(builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
status_out, ev)
# TODO: Could instantiate "printer" instead of showing ops
InteractiveLoop(opts, ex, c_parser, w_parser, line_reader)
status = 0 # TODO: set code
else:
# Parse the whole thing up front
#print('Parsing file')
tlog('ParseWholeFile')
# TODO: Do I need ParseAndEvalLoop? How is it different than
# InteractiveLoop?
try:
node = c_parser.ParseWholeFile()
except util.ParseError as e:
ui.PrettyPrintError(e, arena, sys.stderr)
print('parse error: %s' % e.UserErrorString(), file=sys.stderr)
return 2
else:
# TODO: Remove this older form of error handling.
if not node:
err = c_parser.Error()
ui.PrintErrorStack(err, arena, sys.stderr)
return 2 # parse error is code 2
do_exec = True
if opts.fix:
osh2oil.PrintAsOil(arena, node, opts.debug_spans)
do_exec = False
if exec_opts.noexec:
do_exec = False
if exec_opts.noexec or opts.show_ast: # -n shows the AST
if opts.ast_format == 'oheap':
# TODO: Make this a separate flag?
if sys.stdout.isatty():
raise RuntimeError(
'ERROR: Not dumping binary data to a TTY.')
f = sys.stdout
enc = encode.Params()
out = encode.BinOutput(f)
encode.EncodeRoot(node, enc, out)
else: # text output
f = sys.stdout
if opts.ast_format in ('text', 'abbrev-text'):
ast_f = fmt.DetectConsoleOutput(f)
elif opts.ast_format in ('html', 'abbrev-html'):
ast_f = fmt.HtmlOutput(f)
else:
raise AssertionError
abbrev_hook = (ast.AbbreviateNodes
if 'abbrev-' in opts.ast_format else None)
tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
#util.log("Execution skipped because 'noexec' is on ")
status = 0
if do_exec:
tlog('Execute(node)')
status = ex.Execute(node)
else:
status = 0
return status
