def PrintDebugString(self, debug_f): from osh import ast_lib #debug_f.log('trail = %s', trail) debug_f.log(' words:') for w in self.words: ast_lib.PrettyPrint(w, f=debug_f) debug_f.log('') debug_f.log(' redirects:') for r in self.redirects: ast_lib.PrettyPrint(r, f=debug_f) debug_f.log('') debug_f.log(' tokens:') for p in self.tokens: ast_lib.PrettyPrint(p, f=debug_f) debug_f.log('')
def _assertReadWordFailure(test, word_str): print('\n---', word_str) w_parser = InitWordParser(word_str) w = w_parser.ReadWord(lex_mode_e.OUTER) if w: ast_lib.PrettyPrint(w) test.fail('Expected a parser error, got %r' % w) else: print(w_parser.Error())
def _assertReadWordFailure(test, word_str): print('\n---', word_str) w_parser = InitWordParser(word_str) try: w = w_parser.ReadWord(lex_mode_e.OUTER) except util.ParseError as e: print(e) else: ast_lib.PrettyPrint(w) test.fail('Expected a parser error, got %r' % w)
def _assert_ParseCommandListError(test, code_str): arena, c_parser = InitCommandParser(code_str) try: node = c_parser._ParseCommandLine() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stdout) else: print('UNEXPECTED:') ast_lib.PrettyPrint(node) test.fail("Expected %r to fail" % code_str)
def _assertParseCommandListError(test, code_str): arena, c_parser = InitCommandParser(code_str) node = c_parser.ParseCommandLine() if node: print('UNEXPECTED:') ast_lib.PrettyPrint(node) test.fail("Expected %r to fail" % code_str) return err = c_parser.Error() #print(err) ui.PrintErrorStack(err, arena, sys.stdout)
def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader): if opts.show_ast: ast_f = fmt.DetectConsoleOutput(sys.stdout) else: ast_f = None status = 0 while True: try: w = c_parser.Peek() except KeyboardInterrupt: print('Ctrl-C') break if w is None: raise RuntimeError('Failed parse: %s' % c_parser.Error()) c_id = word.CommandId(w) if c_id == Id.Op_Newline: print('nothing to execute') elif c_id == Id.Eof_Real: print('EOF') break else: node = c_parser.ParseCommandLine() # TODO: Need an error for an empty command, which we ignore? GetLine # could do that in the first position? # ParseSimpleCommand fails with '\n' token? if not node: # TODO: PrintError here raise RuntimeError('failed parse: %s' % c_parser.Error()) if ast_f: ast_lib.PrettyPrint(node) status, is_control_flow = ex.ExecuteAndCatch(node) if is_control_flow: # exit or return break if opts.print_status: print('STATUS', repr(status)) # Reset prompt to PS1. line_reader.Reset() # Reset internal newline state. # NOTE: It would actually be correct to reinitialize all objects (except # Env) on every iteration. But we know that the w_parser is the only thing # that needs to be reset, for now. w_parser.Reset() c_parser.Reset() return status
def _assertReadWordWithArena(test, word_str): print('\n---', word_str) arena, w_parser = _InitWordParserWithArena(word_str) w = w_parser.ReadWord(lex_mode_e.OUTER) assert w is not None ast_lib.PrettyPrint(w) # Next word must be Eof_Real w2 = w_parser.ReadWord(lex_mode_e.OUTER) test.assertTrue( test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')), w2), w2) return arena, w
def _assertReadWordWithArena(test, word_str): print('\n---', word_str) arena = test_lib.MakeArena('word_parse_test.py') w_parser = _InitWordParser(word_str, arena=arena) w = w_parser.ReadWord(lex_mode_e.Outer) assert w is not None ast_lib.PrettyPrint(w) # Next word must be Eof_Real w2 = w_parser.ReadWord(lex_mode_e.Outer) test.assertTrue( test_lib.TokenWordsEqual( osh_word.TokenWord(syntax_asdl.token(Id.Eof_Real, '')), w2), w2) return arena, w
def _assertParseMethod(test, code_str, method, expect_success=True): arena, c_parser = InitCommandParser(code_str) m = getattr(c_parser, method) node = m() if node: ast_lib.PrettyPrint(node) if not expect_success: test.fail('Expected %r to fail ' % code_str) else: # TODO: Could copy PrettyPrintError from pysh.py err = c_parser.Error() print(err) ui.PrintErrorStack(err, arena, sys.stdout) if expect_success: test.fail('%r failed' % code_str) return node
def _assertReadWordWithArena(test, word_str): print('\n---', word_str) arena, w_parser = _InitWordParserWithArena(word_str) w = w_parser.ReadWord(lex_mode_e.OUTER) if w: ast_lib.PrettyPrint(w) else: err = w_parser.Error() test.fail("Couldn't parse %r: %s" % (word_str, err)) # Next word must be Eof_Real w2 = w_parser.ReadWord(lex_mode_e.OUTER) test.assertTrue( test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')), w2), w2) return arena, w
def testReadArith(self): CASES = [ '1 + 2', 'a + b', '$a * $b', '${a} * ${b}', '$(echo 1) * $(echo 2)', '`echo 1` + 2', '$((1 + 2)) * $((3 + 4))', "'single quoted'", # Allowed by oil but not bash '"${a}" + "${b}"', # Ditto '$# + $$', # This doesn't work but does in bash -- should be 15 #'$(( $(echo 1)$(echo 2) + 3 ))', '$(( x[0] < 5 ))', '$(( ++i ))', '$(( i++ ))', '$(( x -= 1))', '$(( x |= 1))', '$(( x[0] = 1 ))', '$(( 1 | 0 ))', '$((0x$size))', ] for expr in CASES: print('---') print(expr) print() w_parser = InitWordParser(expr) w_parser._Next(lex_mode_e.ARITH) # Can we remove this requirement? while True: w = w_parser.ReadWord(lex_mode_e.ARITH) if not w: err = w_parser.Error() print('ERROR', err) self.fail(err) break ast_lib.PrettyPrint(w) if word.CommandId(w) in (Id.Eof_Real, Id.Unknown_Tok): break
def _assertParseMethod(test, code_str, method, expect_success=True): arena, c_parser = InitCommandParser(code_str) m = getattr(c_parser, method) try: if method == 'ParseSimpleCommand': node = m([]) # required cur_aliases arg else: node = m() except util.ParseError as e: ui.PrettyPrintError(e, arena, sys.stdout) if expect_success: test.fail('%r failed' % code_str) node = None else: ast_lib.PrettyPrint(node) if not expect_success: test.fail('Expected %r to fail ' % code_str) return node
def testRead(self): CASES = [ 'ls "foo"', '$(( 1 + 2 ))', '$(echo $(( 1 )) )', # OLD BUG: arith sub within command sub 'echo ${#array[@]} b', # Had a bug here 'echo $(( ${#array[@]} ))', # Bug here # Had a bug: unary minus #'${mounted_disk_regex:0:-1}', 'echo ${@%suffix}', # had a bug here '${@}', 'echo ${var,,}', 'echo ${var,,?}', # Line continuation tests '${\\\nfoo}', # VS_1 '${foo\\\n}', # VS_2 '${foo#\\\nyo}', # VS_ARG_UNQ '"${foo#\\\nyo}"', # VS_ARG_DQ ] for expr in CASES: print('---') print(expr) print() w_parser = InitWordParser(expr) while True: w = w_parser.ReadWord(lex_mode_e.OUTER) if w is None: e = w_parser.Error() print('Error in word parser: %s' % e) self.fail(e) ast_lib.PrettyPrint(w) if word.CommandId(w) == Id.Eof_Real: break
def _MaybeExpandAliases(self, words, cur_aliases): """Try to expand aliases. Our implementation of alias has two design choices: - Where to insert it in parsing. We do it at the end of ParseSimpleCommand. - What grammar rule to parse the expanded alias buffer with. In our case it's ParseCommand(). This doesn't quite match what other shells do, but I can't figure out a better places. Most test cases pass, except for ones like: alias LBRACE='{' LBRACE echo one; echo two; } alias MULTILINE='echo 1 echo 2 echo 3' MULTILINE NOTE: dash handles aliases in a totally diferrent way. It has a global variable checkkwd in parser.c. It assigns it all over the grammar, like this: checkkwd = CHKNL | CHKKWD | CHKALIAS; The readtoken() function checks (checkkwd & CHKALIAS) and then calls lookupalias(). This seems to provide a consistent behavior among shells, but it's less modular and testable. Bash also uses a global 'parser_state & PST_ALEXPNEXT'. Returns: A command node if any aliases were expanded, or None otherwise. """ # The last char that we might parse. right_spid = word.RightMostSpanForWord(words[-1]) first_word_str = None # for error message expanded = [] i = 0 n = len(words) while i < n: w = words[i] ok, word_str, quoted = word.StaticEval(w) if not ok or quoted: break alias_exp = self.aliases.get(word_str) if alias_exp is None: break # Prevent infinite loops. This is subtle: we want to prevent infinite # expansion of alias echo='echo x'. But we don't want to prevent # expansion of the second word in 'echo echo', so we add 'i' to # "cur_aliases". if (word_str, i) in cur_aliases: break if i == 0: first_word_str = word_str # for error message #log('%r -> %r', word_str, alias_exp) cur_aliases.append((word_str, i)) expanded.append(alias_exp) i += 1 if not alias_exp.endswith(' '): # alias e='echo [ ' is the same expansion as # alias e='echo [' # The trailing space indicates whether we should continue to expand # aliases; it's not part of it. expanded.append(' ') break # No more expansions if not expanded: # No expansions; caller does parsing. return None # We got some expansion. Now copy the rest of the words. # We need each NON-REDIRECT word separately! For example: # $ echo one >out two # dash/mksh/zsh go beyond the first redirect! while i < n: w = words[i] left_spid = word.LeftMostSpanForWord(w) right_spid = word.RightMostSpanForWord(w) # Adapted from tools/osh2oil.py Cursor.PrintUntil for span_id in xrange(left_spid, right_spid + 1): span = self.arena.GetLineSpan(span_id) line = self.arena.GetLine(span.line_id) piece = line[span.col : span.col + span.length] expanded.append(piece) expanded.append(' ') # Put space back between words. i += 1 code_str = ''.join(expanded) lines = code_str.splitlines(True) # Keep newlines line_info = [] # TODO: Add location information self.arena.PushSource( '<expansion of alias %r at line %d of %s>' % (first_word_str, -1, 'TODO')) try: for i, line in enumerate(lines): line_id = self.arena.AddLine(line, i+1) line_info.append((line_id, line, 0)) finally: self.arena.PopSource() line_reader = reader.VirtualLineReader(line_info, self.arena) cp = self.parse_ctx.MakeOshParser(line_reader) try: node = cp.ParseCommand(cur_aliases=cur_aliases) except util.ParseError as e: # Failure to parse alias expansion is a fatal error # We don't need more handling here/ raise if 0: log('AFTER expansion:') from osh import ast_lib ast_lib.PrettyPrint(node) return node