def PrintDebugString(self, debug_f):
from osh import ast_lib
#debug_f.log('trail = %s', trail)
debug_f.log(' words:')
for w in self.words:
ast_lib.PrettyPrint(w, f=debug_f)
debug_f.log('')
debug_f.log(' redirects:')
for r in self.redirects:
ast_lib.PrettyPrint(r, f=debug_f)
debug_f.log('')
debug_f.log(' tokens:')
for p in self.tokens:
ast_lib.PrettyPrint(p, f=debug_f)
debug_f.log('')
def _assertReadWordFailure(test, word_str):
print('\n---', word_str)
w_parser = InitWordParser(word_str)
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w:
ast_lib.PrettyPrint(w)
test.fail('Expected a parser error, got %r' % w)
else:
print(w_parser.Error())
def _assertReadWordFailure(test, word_str):
print('\n---', word_str)
w_parser = InitWordParser(word_str)
try:
w = w_parser.ReadWord(lex_mode_e.OUTER)
except util.ParseError as e:
print(e)
else:
ast_lib.PrettyPrint(w)
test.fail('Expected a parser error, got %r' % w)
def _assert_ParseCommandListError(test, code_str):
arena, c_parser = InitCommandParser(code_str)
try:
node = c_parser._ParseCommandLine()
except util.ParseError as e:
ui.PrettyPrintError(e, arena, sys.stdout)
else:
print('UNEXPECTED:')
ast_lib.PrettyPrint(node)
test.fail("Expected %r to fail" % code_str)
def _assertParseCommandListError(test, code_str):
arena, c_parser = InitCommandParser(code_str)
node = c_parser.ParseCommandLine()
if node:
print('UNEXPECTED:')
ast_lib.PrettyPrint(node)
test.fail("Expected %r to fail" % code_str)
return
err = c_parser.Error()
#print(err)
ui.PrintErrorStack(err, arena, sys.stdout)
def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader):
if opts.show_ast:
ast_f = fmt.DetectConsoleOutput(sys.stdout)
else:
ast_f = None
status = 0
while True:
try:
w = c_parser.Peek()
except KeyboardInterrupt:
print('Ctrl-C')
break
if w is None:
raise RuntimeError('Failed parse: %s' % c_parser.Error())
c_id = word.CommandId(w)
if c_id == Id.Op_Newline:
print('nothing to execute')
elif c_id == Id.Eof_Real:
print('EOF')
break
else:
node = c_parser.ParseCommandLine()
# TODO: Need an error for an empty command, which we ignore? GetLine
# could do that in the first position?
# ParseSimpleCommand fails with '\n' token?
if not node:
# TODO: PrintError here
raise RuntimeError('failed parse: %s' % c_parser.Error())
if ast_f:
ast_lib.PrettyPrint(node)
status, is_control_flow = ex.ExecuteAndCatch(node)
if is_control_flow: # exit or return
break
if opts.print_status:
print('STATUS', repr(status))
# Reset prompt to PS1.
line_reader.Reset()
# Reset internal newline state.
# NOTE: It would actually be correct to reinitialize all objects (except
# Env) on every iteration. But we know that the w_parser is the only thing
# that needs to be reset, for now.
w_parser.Reset()
c_parser.Reset()
return status
def _assertReadWordWithArena(test, word_str):
print('\n---', word_str)
arena, w_parser = _InitWordParserWithArena(word_str)
w = w_parser.ReadWord(lex_mode_e.OUTER)
assert w is not None
ast_lib.PrettyPrint(w)
# Next word must be Eof_Real
w2 = w_parser.ReadWord(lex_mode_e.OUTER)
test.assertTrue(
test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')),
w2), w2)
return arena, w
def _assertReadWordWithArena(test, word_str):
print('\n---', word_str)
arena = test_lib.MakeArena('word_parse_test.py')
w_parser = _InitWordParser(word_str, arena=arena)
w = w_parser.ReadWord(lex_mode_e.Outer)
assert w is not None
ast_lib.PrettyPrint(w)
# Next word must be Eof_Real
w2 = w_parser.ReadWord(lex_mode_e.Outer)
test.assertTrue(
test_lib.TokenWordsEqual(
osh_word.TokenWord(syntax_asdl.token(Id.Eof_Real, '')), w2), w2)
return arena, w
def _assertParseMethod(test, code_str, method, expect_success=True):
arena, c_parser = InitCommandParser(code_str)
m = getattr(c_parser, method)
node = m()
if node:
ast_lib.PrettyPrint(node)
if not expect_success:
test.fail('Expected %r to fail ' % code_str)
else:
# TODO: Could copy PrettyPrintError from pysh.py
err = c_parser.Error()
print(err)
ui.PrintErrorStack(err, arena, sys.stdout)
if expect_success:
test.fail('%r failed' % code_str)
return node
def _assertReadWordWithArena(test, word_str):
print('\n---', word_str)
arena, w_parser = _InitWordParserWithArena(word_str)
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w:
ast_lib.PrettyPrint(w)
else:
err = w_parser.Error()
test.fail("Couldn't parse %r: %s" % (word_str, err))
# Next word must be Eof_Real
w2 = w_parser.ReadWord(lex_mode_e.OUTER)
test.assertTrue(
test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')),
w2), w2)
return arena, w
def testReadArith(self):
CASES = [
'1 + 2',
'a + b',
'$a * $b',
'${a} * ${b}',
'$(echo 1) * $(echo 2)',
'`echo 1` + 2',
'$((1 + 2)) * $((3 + 4))',
"'single quoted'", # Allowed by oil but not bash
'"${a}" + "${b}"', # Ditto
'$# + $$',
# This doesn't work but does in bash -- should be 15
#'$(( $(echo 1)$(echo 2) + 3 ))',
'$(( x[0] < 5 ))',
'$(( ++i ))',
'$(( i++ ))',
'$(( x -= 1))',
'$(( x |= 1))',
'$(( x[0] = 1 ))',
'$(( 1 | 0 ))',
'$((0x$size))',
]
for expr in CASES:
print('---')
print(expr)
print()
w_parser = InitWordParser(expr)
w_parser._Next(lex_mode_e.ARITH) # Can we remove this requirement?
while True:
w = w_parser.ReadWord(lex_mode_e.ARITH)
if not w:
err = w_parser.Error()
print('ERROR', err)
self.fail(err)
break
ast_lib.PrettyPrint(w)
if word.CommandId(w) in (Id.Eof_Real, Id.Unknown_Tok):
break
def _assertParseMethod(test, code_str, method, expect_success=True):
arena, c_parser = InitCommandParser(code_str)
m = getattr(c_parser, method)
try:
if method == 'ParseSimpleCommand':
node = m([]) # required cur_aliases arg
else:
node = m()
except util.ParseError as e:
ui.PrettyPrintError(e, arena, sys.stdout)
if expect_success:
test.fail('%r failed' % code_str)
node = None
else:
ast_lib.PrettyPrint(node)
if not expect_success:
test.fail('Expected %r to fail ' % code_str)
return node
def testRead(self):
CASES = [
'ls "foo"',
'$(( 1 + 2 ))',
'$(echo $(( 1 )) )', # OLD BUG: arith sub within command sub
'echo ${#array[@]} b', # Had a bug here
'echo $(( ${#array[@]} ))', # Bug here
# Had a bug: unary minus
#'${mounted_disk_regex:0:-1}',
'echo ${@%suffix}', # had a bug here
'${@}',
'echo ${var,,}',
'echo ${var,,?}',
# Line continuation tests
'${\\\nfoo}', # VS_1
'${foo\\\n}', # VS_2
'${foo#\\\nyo}', # VS_ARG_UNQ
'"${foo#\\\nyo}"', # VS_ARG_DQ
]
for expr in CASES:
print('---')
print(expr)
print()
w_parser = InitWordParser(expr)
while True:
w = w_parser.ReadWord(lex_mode_e.OUTER)
if w is None:
e = w_parser.Error()
print('Error in word parser: %s' % e)
self.fail(e)
ast_lib.PrettyPrint(w)
if word.CommandId(w) == Id.Eof_Real:
break
def _MaybeExpandAliases(self, words, cur_aliases):
"""Try to expand aliases.
Our implementation of alias has two design choices:
- Where to insert it in parsing. We do it at the end of ParseSimpleCommand.
- What grammar rule to parse the expanded alias buffer with. In our case
it's ParseCommand().
This doesn't quite match what other shells do, but I can't figure out a
better places.
Most test cases pass, except for ones like:
alias LBRACE='{'
LBRACE echo one; echo two; }
alias MULTILINE='echo 1
echo 2
echo 3'
MULTILINE
NOTE: dash handles aliases in a totally diferrent way. It has a global
variable checkkwd in parser.c. It assigns it all over the grammar, like
this:
checkkwd = CHKNL | CHKKWD | CHKALIAS;
The readtoken() function checks (checkkwd & CHKALIAS) and then calls
lookupalias(). This seems to provide a consistent behavior among shells,
but it's less modular and testable.
Bash also uses a global 'parser_state & PST_ALEXPNEXT'.
Returns:
A command node if any aliases were expanded, or None otherwise.
"""
# The last char that we might parse.
right_spid = word.RightMostSpanForWord(words[-1])
first_word_str = None # for error message
expanded = []
i = 0
n = len(words)
while i < n:
w = words[i]
ok, word_str, quoted = word.StaticEval(w)
if not ok or quoted:
break
alias_exp = self.aliases.get(word_str)
if alias_exp is None:
break
# Prevent infinite loops. This is subtle: we want to prevent infinite
# expansion of alias echo='echo x'. But we don't want to prevent
# expansion of the second word in 'echo echo', so we add 'i' to
# "cur_aliases".
if (word_str, i) in cur_aliases:
break
if i == 0:
first_word_str = word_str # for error message
#log('%r -> %r', word_str, alias_exp)
cur_aliases.append((word_str, i))
expanded.append(alias_exp)
i += 1
if not alias_exp.endswith(' '):
# alias e='echo [ ' is the same expansion as
# alias e='echo ['
# The trailing space indicates whether we should continue to expand
# aliases; it's not part of it.
expanded.append(' ')
break # No more expansions
if not expanded: # No expansions; caller does parsing.
return None
# We got some expansion. Now copy the rest of the words.
# We need each NON-REDIRECT word separately! For example:
# $ echo one >out two
# dash/mksh/zsh go beyond the first redirect!
while i < n:
w = words[i]
left_spid = word.LeftMostSpanForWord(w)
right_spid = word.RightMostSpanForWord(w)
# Adapted from tools/osh2oil.py Cursor.PrintUntil
for span_id in xrange(left_spid, right_spid + 1):
span = self.arena.GetLineSpan(span_id)
line = self.arena.GetLine(span.line_id)
piece = line[span.col : span.col + span.length]
expanded.append(piece)
expanded.append(' ') # Put space back between words.
i += 1
code_str = ''.join(expanded)
lines = code_str.splitlines(True) # Keep newlines
line_info = []
# TODO: Add location information
self.arena.PushSource(
'<expansion of alias %r at line %d of %s>' %
(first_word_str, -1, 'TODO'))
try:
for i, line in enumerate(lines):
line_id = self.arena.AddLine(line, i+1)
line_info.append((line_id, line, 0))
finally:
self.arena.PopSource()
line_reader = reader.VirtualLineReader(line_info, self.arena)
cp = self.parse_ctx.MakeOshParser(line_reader)
try:
node = cp.ParseCommand(cur_aliases=cur_aliases)
except util.ParseError as e:
# Failure to parse alias expansion is a fatal error
# We don't need more handling here/
raise
if 0:
log('AFTER expansion:')
from osh import ast_lib
ast_lib.PrettyPrint(node)
return node