Esempio n. 1
0
    def _ReadArithWord(self):
        """Helper function for ReadArithWord."""
        #assert self.token_type != Id.Undefined_Tok
        self._Peek()
        #print('_ReadArithWord', self.cur_token)

        if self.token_kind == Kind.Unknown:
            self.AddErrorContext("Unknown token in arith context: %s",
                                 self.cur_token,
                                 token=self.cur_token)
            return None, False

        elif self.token_kind == Kind.Eof:
            # Just return EOF token
            w = ast.TokenWord(self.cur_token)
            return w, False
            #self.AddErrorContext("Unexpected EOF in arith context: %s",
            #    self.cur_token, token=self.cur_token)
            #return None, False

        elif self.token_kind == Kind.Ignored:
            # Space should be ignored.  TODO: change this to SPACE_SPACE and
            # SPACE_NEWLINE?  or SPACE_TOK.
            self._Next(lex_mode_e.ARITH)
            return None, True  # Tell wrapper to try again

        elif self.token_kind in (Kind.Arith, Kind.Right):
            # Id.Right_ArithSub IS just a normal token, handled by ArithParser
            self._Next(lex_mode_e.ARITH)
            w = ast.TokenWord(self.cur_token)
            return w, False

        elif self.token_kind in (Kind.Lit, Kind.Left):
            w = self._ReadCompoundWord(lex_mode=lex_mode_e.ARITH)
            if not w:
                return None, True
            return w, False

        elif self.token_kind == Kind.VSub:
            part = ast.SimpleVarSub(self.cur_token)
            self._Next(lex_mode_e.ARITH)
            w = ast.CompoundWord([part])
            return w, False

        else:
            self._BadToken("Unexpected token parsing arith sub: %s",
                           self.cur_token)
            return None, False

        raise AssertionError("Shouldn't get here")
Esempio n. 2
0
    def _ReadArithWord(self):
        """Helper function for ReadArithWord."""
        self._Peek()

        if self.token_kind == Kind.Unknown:
            p_die('Unexpected token in arithmetic context',
                  token=self.cur_token)

        elif self.token_kind == Kind.Eof:
            # Just return EOF token
            w = ast.TokenWord(self.cur_token)
            return w, False

        elif self.token_kind == Kind.Ignored:
            # Space should be ignored.  TODO: change this to SPACE_SPACE and
            # SPACE_NEWLINE?  or SPACE_TOK.
            self._Next(lex_mode_e.ARITH)
            return None, True  # Tell wrapper to try again

        elif self.token_kind in (Kind.Arith, Kind.Right):
            # Id.Right_ArithSub IS just a normal token, handled by ArithParser
            self._Next(lex_mode_e.ARITH)
            w = ast.TokenWord(self.cur_token)
            return w, False

        elif self.token_kind in (Kind.Lit, Kind.Left):
            w = self._ReadCompoundWord(lex_mode=lex_mode_e.ARITH)
            return w, False

        elif self.token_kind == Kind.VSub:
            part = ast.SimpleVarSub(self.cur_token)
            self._Next(lex_mode_e.ARITH)
            w = ast.CompoundWord([part])
            return w, False

        else:
            assert False, ("Unexpected token parsing arith sub: %s" %
                           self.cur_token)

        raise AssertionError("Shouldn't get here")
Esempio n. 3
0
def _assertReadWordWithArena(test, word_str):
    print('\n---', word_str)
    arena, w_parser = _InitWordParserWithArena(word_str)
    w = w_parser.ReadWord(lex_mode_e.OUTER)
    assert w is not None
    ast_lib.PrettyPrint(w)

    # Next word must be Eof_Real
    w2 = w_parser.ReadWord(lex_mode_e.OUTER)
    test.assertTrue(
        test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')),
                                 w2), w2)

    return arena, w
Esempio n. 4
0
    def testMultiLine(self):
        w_parser = InitWordParser("""\
ls foo

# Multiple newlines and comments should be ignored

ls bar
""")
        print('--MULTI')
        w = w_parser.ReadWord(lex_mode_e.OUTER)
        parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls'))]
        test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w)

        w = w_parser.ReadWord(lex_mode_e.OUTER)
        parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'foo'))]
        test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w)

        w = w_parser.ReadWord(lex_mode_e.OUTER)
        t = ast.token(Id.Op_Newline, '\n')
        test_lib.AssertAsdlEqual(self, ast.TokenWord(t), w)

        w = w_parser.ReadWord(lex_mode_e.OUTER)
        parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'ls'))]
        test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w)

        w = w_parser.ReadWord(lex_mode_e.OUTER)
        parts = [ast.LiteralPart(ast.token(Id.Lit_Chars, 'bar'))]
        test_lib.AssertAsdlEqual(self, ast.CompoundWord(parts), w)

        w = w_parser.ReadWord(lex_mode_e.OUTER)
        t = ast.token(Id.Op_Newline, '\n')
        test_lib.AssertAsdlEqual(self, ast.TokenWord(t), w)

        w = w_parser.ReadWord(lex_mode_e.OUTER)
        t = ast.token(Id.Eof_Real, '')
        test_lib.AssertAsdlEqual(self, ast.TokenWord(t), w)
Esempio n. 5
0
def _assertReadWordWithArena(test, word_str):
    print('\n---', word_str)
    arena, w_parser = _InitWordParserWithArena(word_str)
    w = w_parser.ReadWord(lex_mode_e.OUTER)
    if w:
        ast_lib.PrettyPrint(w)
    else:
        err = w_parser.Error()
        test.fail("Couldn't parse %r: %s" % (word_str, err))

    # Next word must be Eof_Real
    w2 = w_parser.ReadWord(lex_mode_e.OUTER)
    test.assertTrue(
        test_lib.TokenWordsEqual(ast.TokenWord(ast.token(Id.Eof_Real, '')),
                                 w2), w2)

    return arena, w
Esempio n. 6
0
    def _ReadWord(self, lex_mode):
        """Helper function for Read().

    Returns:
      2-tuple (word, need_more)
        word: Word, or None if there was an error, or need_more is set
        need_more: True if the caller should call us again
    """
        #print('_Read', lex_mode, self.cur_token)
        self._Peek()

        if self.token_kind == Kind.Eof:
            # No advance
            return ast.TokenWord(self.cur_token), False

        # Allow Arith for ) at end of for loop?
        elif self.token_kind in (Kind.Op, Kind.Redir, Kind.Arith):
            self._Next(lex_mode)
            if self.token_type == Id.Op_Newline:
                if self.cursor_was_newline:
                    #print('SKIP(nl)', self.cur_token)
                    return None, True

            return ast.TokenWord(self.cur_token), False

        elif self.token_kind == Kind.Right:
            #print('WordParser.Read: Kind.Right', self.cur_token)
            if self.token_type not in (Id.Right_Subshell, Id.Right_FuncDef,
                                       Id.Right_CasePat,
                                       Id.Right_ArrayLiteral):
                raise AssertionError(self.cur_token)

            self._Next(lex_mode)
            return ast.TokenWord(self.cur_token), False

        elif self.token_kind in (Kind.Ignored, Kind.WS):
            self._Next(lex_mode)
            return None, True  # tell Read() to try again

        elif self.token_kind in (Kind.VSub, Kind.Lit, Kind.Left, Kind.KW,
                                 Kind.Assign, Kind.ControlFlow, Kind.BoolUnary,
                                 Kind.BoolBinary, Kind.ExtGlob):
            # We're beginning a word.  If we see Id.Lit_Pound, change to
            # lex_mode_e.COMMENT and read until end of line.  (TODO: How to add
            # comments to AST?)

            # TODO: Can we do the same thing for Tilde here?  Enter a state where we
            # look for / too.
            if self.token_type == Id.Lit_Pound:
                self._Next(lex_mode_e.COMMENT)
                self._Peek()

                # NOTE: The # could be the last character in the file.  It can't be
                # Eof_{RParen,Backtick} because #) and #` are comments.
                assert self.token_type in (Id.Ignored_Comment, Id.Eof_Real), \
                    self.cur_token

                # The next iteration will go into Kind.Ignored and set lex state to
                # lex_mode_e.OUTER/etc.
                return None, True  # tell Read() to try again after comment

            else:
                w = self._ReadCompoundWord(lex_mode=lex_mode)
                if not w:
                    self.AddErrorContext('Error reading command word',
                                         token=self.cur_token)
                    return None, False
                return w, False

        else:
            raise AssertionError('Unhandled: %s (%s)' %
                                 (self.cur_token, self.token_kind))

        raise AssertionError("Shouldn't get here")