示例#1
0
  def _DictPair(self, p_node):
    # type: (PNode) -> Tuple[expr_t, expr_t]
    """
    dict_pair: (
      Expr_Name [':' test] |
      '[' testlist ']' ':' test
    )
    """
    assert p_node.typ == grammar_nt.dict_pair

    children = p_node.children
    typ = children[0].typ

    if ISNONTERMINAL(typ):  # for sq_string
      # Note: Could inline these cases instead of going through self.Expr.
      if typ == grammar_nt.sq_string:
        key = self.Expr(children[0])  # type: expr_t
      elif typ == grammar_nt.dq_string:
        key = self.Expr(children[0])

      value = self.Expr(children[2])
      return key, value

    tok0 = children[0].tok
    id_ = tok0.id

    if id_ == Id.Expr_Name:
      key = expr.Const(tok0)
      if len(children) >= 3:
        value = self.Expr(children[2])
      else:
        value = expr.Implicit()

    if id_ == Id.Op_LBracket:  # {[x+y]: 'val'}
      key = self.Expr(children[1])
      value = self.Expr(children[4])
      return key, value

    return key, value
示例#2
0
  def Expr(self, pnode):
    # type: (PNode) -> expr_t
    """Transform expressions (as opposed to statements)."""
    typ = pnode.typ
    tok = pnode.tok
    children = pnode.children

    if ISNONTERMINAL(typ):

      #
      # Oil Entry Points / Additions
      #

      if typ == grammar_nt.oil_expr:  # for if/while
        # oil_expr: '(' testlist ')'
        return self.Expr(children[1])

      if typ == grammar_nt.command_expr:
        # return_expr: testlist end_stmt
        return self.Expr(children[0])

      #
      # Python-like Expressions / Operators
      #

      if typ == grammar_nt.atom:
        if len(children) == 1:
          return self.Expr(children[0])
        return self._Atom(children)

      if typ == grammar_nt.testlist:
        # testlist: test (',' test)* [',']
        return self._Tuple(children)

      if typ == grammar_nt.test:
        # test: or_test ['if' or_test 'else' test] | lambdef
        if len(children) == 1:
          return self.Expr(children[0])

        # TODO: Handle lambdef

        test = self.Expr(children[2])
        body = self.Expr(children[0])
        orelse = self.Expr(children[4])
        return expr.IfExp(test, body, orelse)

      if typ == grammar_nt.lambdef:
        # lambdef: '|' [name_type_list] '|' test

        n = len(children)
        if n == 4:
          params = self._NameTypeList(children[1])
        else:
          params = []

        body = self.Expr(children[n-1])
        return expr.Lambda(params, body)

      #
      # Operators with Precedence
      #

      if typ == grammar_nt.or_test:
        # or_test: and_test ('or' and_test)*
        return self._AssocBinary(children)

      if typ == grammar_nt.and_test:
        # and_test: not_test ('and' not_test)*
        return self._AssocBinary(children)

      if typ == grammar_nt.not_test:
        # not_test: 'not' not_test | comparison
        if len(children) == 1:
          return self.Expr(children[0])

        op_tok = children[0].tok  # not
        return expr.Unary(op_tok, self.Expr(children[1]))

      elif typ == grammar_nt.comparison:
        if len(children) == 1:
          return self.Expr(children[0])

        return self._CompareChain(children)

      elif typ == grammar_nt.range_expr:
        n = len(children)
        if n == 1:
          return self.Expr(children[0])

        if n == 3:
          return expr.Range(
              self.Expr(children[0]),
              self.Expr(children[2])
          )

        raise AssertionError(n)

      elif typ == grammar_nt.expr:
        # expr: xor_expr ('|' xor_expr)*
        return self._AssocBinary(children)

      if typ == grammar_nt.xor_expr:
        # xor_expr: and_expr ('xor' and_expr)*
        return self._AssocBinary(children)

      if typ == grammar_nt.and_expr:  # a & b
        # and_expr: shift_expr ('&' shift_expr)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.shift_expr:
        # shift_expr: arith_expr (('<<'|'>>') arith_expr)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.arith_expr:
        # arith_expr: term (('+'|'-') term)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.term:
        # term: factor (('*'|'/'|'div'|'mod') factor)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.factor:
        # factor: ('+'|'-'|'~') factor | power
        # the power would have already been reduced
        if len(children) == 1:
          return self.Expr(children[0])

        assert len(children) == 2
        op = children[0]
        e = children[1]

        assert isinstance(op.tok, Token)
        return expr.Unary(op.tok, self.Expr(e))

      elif typ == grammar_nt.power:
        # power: atom trailer* ['^' factor]

        node = self.Expr(children[0])
        if len(children) == 1:  # No trailers
          return node

        n = len(children)
        i = 1
        while i < n and ISNONTERMINAL(children[i].typ):
          node = self._Trailer(node, children[i])
          i += 1

        if i != n:  # ['^' factor]
          op_tok = children[i].tok
          assert op_tok.id == Id.Arith_Caret, op_tok
          factor = self.Expr(children[i+1])
          node = expr.Binary(op_tok, node, factor)

        return node

      elif typ == grammar_nt.array_literal:
        left_tok = children[0].tok

        items = [self._ArrayItem(p) for p in children[1:-1]]
        return expr.ArrayLiteral(left_tok, items)

      elif typ == grammar_nt.oil_expr_sub:
        return self.Expr(children[0])

      #
      # Oil Lexer Modes
      #

      elif typ == grammar_nt.sh_array_literal:
        return cast(sh_array_literal, children[1].tok)

      elif typ == grammar_nt.sh_command_sub:
        return cast(command_sub, children[1].tok)

      elif typ == grammar_nt.braced_var_sub:
        return cast(braced_var_sub, children[1].tok)

      elif typ == grammar_nt.dq_string:
        return cast(double_quoted, children[1].tok)

      elif typ == grammar_nt.sq_string:
        return cast(single_quoted, children[1].tok)

      elif typ == grammar_nt.simple_var_sub:
        return simple_var_sub(children[0].tok)

      else:
        nt_name = self.number2symbol[typ]
        raise AssertionError(
            "PNode type %d (%s) wasn't handled" % (typ, nt_name))

    else:  # Terminals should have a token
      id_ = tok.id

      if id_ == Id.Expr_Name:
        return expr.Var(tok)

      if id_ in (
          Id.Expr_DecInt, Id.Expr_BinInt, Id.Expr_OctInt, Id.Expr_HexInt,
          Id.Expr_Float):
        return expr.Const(tok)

      if id_ in (Id.Expr_Null, Id.Expr_True, Id.Expr_False):
        return expr.Const(tok)

      raise NotImplementedError(Id_str(id_))
示例#3
0
    def Expr(self, pnode):
        # type: (PNode) -> expr_t
        """Transform expressions (as opposed to statements)."""
        typ = pnode.typ
        tok = pnode.tok
        children = pnode.children

        if ISNONTERMINAL(typ):
            c = '-' if not children else len(children)
            #log('non-terminal %s %s', nt_name, c)

            if typ == grammar_nt.oil_expr:  # for if/while
                # oil_expr: '(' testlist ')'
                return self.Expr(children[1])

            if typ == grammar_nt.return_expr:  # for if/while
                # return_expr: testlist end_stmt
                return self.Expr(children[0])

            if typ == grammar_nt.lvalue_list:
                return self._AssocBinary(children)

            if typ == grammar_nt.atom:
                return self.atom(children)

            if typ == grammar_nt.eval_input:
                # testlist_input: testlist NEWLINE* ENDMARKER
                return self.Expr(children[0])

            if typ == grammar_nt.testlist:
                # testlist: test (',' test)* [',']
                return self._AssocBinary(children)

            elif typ == grammar_nt.arith_expr:
                # expr: term (('+'|'-') term)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.term:
                # term: factor (('*'|'/'|'div'|'mod') factor)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.expr:
                # expr: xor_expr ('|' xor_expr)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.shift_expr:
                # shift_expr: arith_expr (('<<'|'>>') arith_expr)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.comparison:
                # comparison: expr (comp_op expr)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.factor:
                # factor: ('+'|'-'|'~') factor | power
                # the power would have already been reduced
                assert len(children) == 2, children
                op, e = children
                assert isinstance(op.tok, token)
                return expr.Unary(op.tok, self.Expr(e))

            elif typ == grammar_nt.atom_expr:
                # atom_expr: ['await'] atom trailer*

                # NOTE: This would be shorter in a recursive style.
                base = self.Expr(children[0])
                n = len(children)
                for i in xrange(1, n):
                    pnode = children[i]
                    tok = pnode.tok
                    base = self.trailer(base, pnode)

                return base

            elif typ == grammar_nt.power:
                # power: atom_expr ['^' factor]

                # This doesn't repeat, so it doesn't matter if it's left or right
                # associative.
                return self._AssocBinary(children)

            elif typ == grammar_nt.array_literal:
                left_tok = children[0].tok

                # Approximation for now.
                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Lit_Chars
                ]
                items = [expr.Const(t) for t in tokens]  # type: List[expr_t]
                return expr.ArrayLiteral(left_tok, items)

            elif typ == grammar_nt.sh_array_literal:
                left_tok = children[0].tok

                # HACK: When typ is Id.Expr_WordsDummy, the 'tok' field ('opaque')
                # actually has a list of words!
                typ1 = children[1].typ
                assert typ1 == Id.Expr_WordsDummy.enum_id, typ1
                array_words = cast('List[word_t]', children[1].tok)

                return expr.ShellArrayLiteral(left_tok, array_words)

            elif typ == grammar_nt.regex_literal:
                left_tok = children[0].tok

                # Approximation for now.
                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Expr_Name
                ]
                parts = [regex.Var(t) for t in tokens]  # type: List[regex_t]

                return expr.RegexLiteral(left_tok, regex.Concat(parts))

            elif typ == grammar_nt.command_sub:
                left_tok = children[0].tok

                # Approximation for now.
                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Lit_Chars
                ]
                words = [
                    word.Compound([word_part.Literal(t)]) for t in tokens
                ]  # type: List[word_t]
                return expr.CommandSub(left_tok, command.Simple(words))

            elif typ == grammar_nt.sh_command_sub:
                left_tok = children[0].tok

                # HACK: When typ is Id.Expr_CommandDummy, the 'tok' field ('opaque')
                # actually has a word_part.CommandSub!
                typ1 = children[1].typ
                assert typ1 == Id.Expr_CommandDummy.enum_id, typ1
                cs_part = cast(word_part__CommandSub, children[1].tok)

                # Awkward: the schemas are different
                expr_part = expr.CommandSub(cs_part.left_token,
                                            cs_part.command_list)
                expr_part.spids.extend(cs_part.spids)
                return expr_part

            elif typ == grammar_nt.var_sub:
                left_tok = children[0].tok

                return expr.VarSub(left_tok, self.Expr(children[1]))

            elif typ == grammar_nt.dq_string:
                left_tok = children[0].tok

                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Lit_Chars
                ]
                parts2 = [word_part.Literal(t)
                          for t in tokens]  # type: List[word_part_t]
                return expr.DoubleQuoted(left_tok, parts2)

            else:
                nt_name = self.number2symbol[typ]
                raise AssertionError("PNode type %d (%s) wasn't handled" %
                                     (typ, nt_name))

        else:  # Terminals should have a token
            #log('terminal %s', tok)

            if tok.id == Id.Expr_Name:
                return expr.Var(tok)
            elif tok.id == Id.Expr_Digits:
                return expr.Const(tok)

            else:
                raise AssertionError(tok.id)
示例#4
0
  def Expr(self, pnode):
    # type: (PNode) -> expr_t
    """Transform expressions (as opposed to statements)."""
    typ = pnode.typ
    tok = pnode.tok
    children = pnode.children

    if ISNONTERMINAL(typ):

      #
      # Oil Entry Points / Additions
      #

      if typ == grammar_nt.oil_expr:  # for if/while
        # oil_expr: '(' testlist ')'
        return self.Expr(children[1])

      if typ == grammar_nt.return_expr:
        # return_expr: testlist end_stmt
        return self.Expr(children[0])

      if typ == grammar_nt.place_list:
        return self._AssocBinary(children)

      if typ == grammar_nt.place:
        # place: NAME place_trailer*
        if len(pnode.children) == 1:
          return self.Expr(pnode.children[0])
        # TODO: Called _Trailer but don't handle ( )?
        # only [] . -> :: ?
        raise NotImplementedError

      #
      # Python-like Expressions / Operators
      #

      if typ == grammar_nt.atom:
        if len(children) == 1:
          return self.Expr(children[0])
        return self._Atom(children)

      if typ == grammar_nt.testlist:
        # testlist: test (',' test)* [',']
        # We need tuples for Python's 'var a, b = x' and 'for (a, b in x) {'
        return self._Tuple(children)

      if typ == grammar_nt.test:
        # test: or_test ['if' or_test 'else' test] | lambdef
        if len(children) == 1:
          return self.Expr(children[0])

        # TODO: Handle lambdef

        test = self.Expr(children[2])
        body = self.Expr(children[0])
        orelse = self.Expr(children[4])
        return expr.IfExp(test, body, orelse)

      if typ == grammar_nt.test_nocond:
        # test_nocond: or_test | lambdef_nocond
        assert len(children) == 1
        return self.Expr(children[0])

      if typ == grammar_nt.argument:
        # argument: ( test [comp_for] |
        #             test '=' test |
        #             '**' test |
        #             '*' test )
        if len(pnode.children) == 1:
          return self.Expr(children[0])
        # TODO:
        raise NotImplementedError

      if typ == grammar_nt.subscript:
        # subscript: test | [test] ':' [test] [sliceop]
        if len(pnode.children) == 1:
          return self.Expr(children[0])
        # TODO:
        raise NotImplementedError

      if typ == grammar_nt.testlist_comp:
        # testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
        if children[1].typ == grammar_nt.comp_for:
          elt = self.Expr(children[0])
          comp = self._CompFor(children[1])
          return expr.ListComp(elt, [comp])

        # (1,)  (1, 2)  etc.
        if children[1].tok.id == Id.Arith_Comma:
          return self._Tuple(children)
        raise NotImplementedError('testlist_comp')

      elif typ == grammar_nt.exprlist:
        # exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']

        if len(children) == 1:
          return self.Expr(children[0])

        # used in for loop, genexpr.
        # TODO: This sould be placelist?  for x, *y ?
        raise NotImplementedError('exprlist')

      #
      # Operators with Precedence
      #

      if typ == grammar_nt.or_test:
        # or_test: and_test ('or' and_test)*
        return self._AssocBinary(children)

      if typ == grammar_nt.and_test:
        # and_test: not_test ('and' not_test)*
        return self._AssocBinary(children)

      if typ == grammar_nt.not_test:
        # not_test: 'not' not_test | comparison
        if len(children) == 1:
          return self.Expr(children[0])

        op_tok = children[0].tok  # not
        return expr.Unary(op_tok, self.Expr(children[1]))

      elif typ == grammar_nt.comparison:
        if len(children) == 1:
          return self.Expr(children[0])

        return self._CompareChain(children)

      elif typ == grammar_nt.expr:
        # expr: xor_expr ('|' xor_expr)*
        return self._AssocBinary(children)

      if typ == grammar_nt.xor_expr:
        # xor_expr: and_expr ('xor' and_expr)*
        return self._AssocBinary(children)

      if typ == grammar_nt.and_expr:  # a & b
        # and_expr: shift_expr ('&' shift_expr)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.shift_expr:
        # shift_expr: arith_expr (('<<'|'>>') arith_expr)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.arith_expr:
        # arith_expr: term (('+'|'-') term)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.term:
        # term: factor (('*'|'/'|'div'|'mod') factor)*
        return self._AssocBinary(children)

      elif typ == grammar_nt.factor:
        # factor: ('+'|'-'|'~') factor | power
        # the power would have already been reduced
        if len(children) == 1:
          return self.Expr(children[0])
        op, e = children
        assert isinstance(op.tok, token)
        return expr.Unary(op.tok, self.Expr(e))

      elif typ == grammar_nt.power:
        # power: atom trailer* ['^' factor]

        node = self.Expr(children[0])
        if len(children) == 1:  # No trailers
          return node

        n = len(children)
        i = 1
        while i < n and ISNONTERMINAL(children[i].typ):
          node = self._Trailer(node, children[i])
          i += 1

        if i != n:  # ['^' factor]
          op_tok = children[i].tok
          assert op_tok.id == Id.Arith_Caret, op_tok
          factor = self.Expr(children[i+1])
          node = expr.Binary(op_tok, node, factor)

        return node

      #
      # Oil Lexer Modes
      #

      elif typ == grammar_nt.array_literal:
        left_tok = children[0].tok

        # Approximation for now.
        tokens = [
            pnode.tok for pnode in children[1:-1] if pnode.tok.id ==
            Id.Lit_Chars
        ]
        items = [expr.Const(t) for t in tokens]  # type: List[expr_t]
        return expr.ArrayLiteral(left_tok, items)

      elif typ == grammar_nt.sh_array_literal:
        left_tok = children[0].tok

        # HACK: When typ is Id.Expr_CastedDummy, the 'tok' field ('opaque')
        # actually has a list of words!
        typ1 = children[1].typ
        assert typ1 == Id.Expr_CastedDummy.enum_id, typ1
        array_words = cast('List[word_t]', children[1].tok)

        return sh_array_literal(left_tok, array_words)

      elif typ == grammar_nt.sh_command_sub:
        return cast(command_sub, children[1].tok)

      elif typ == grammar_nt.braced_var_sub:
        return cast(braced_var_sub, children[1].tok)

      elif typ == grammar_nt.dq_string:
        return cast(double_quoted, children[1].tok)

      elif typ == grammar_nt.sq_string:
        return cast(single_quoted, children[1].tok)

      elif typ == grammar_nt.simple_var_sub:
        return simple_var_sub(children[0].tok)

      else:
        nt_name = self.number2symbol[typ]
        raise AssertionError(
            "PNode type %d (%s) wasn't handled" % (typ, nt_name))

    else:  # Terminals should have a token
      id_ = tok.id

      if id_ == Id.Expr_Name:
        return expr.Var(tok)

      if id_ in (
          Id.Expr_DecInt, Id.Expr_BinInt, Id.Expr_OctInt, Id.Expr_HexInt,
          Id.Expr_Float):
        return expr.Const(tok)

      if id_ in (Id.Expr_Null, Id.Expr_True, Id.Expr_False):
        return expr.Const(tok)

      from core.meta import IdInstance
      raise NotImplementedError(IdInstance(typ))
示例#5
0
文件: expr_to_ast.py 项目: mrshu/oil
    def Expr(self, pnode):
        # type: (PNode) -> expr_t
        """Walk the homogeneous parse tree and create a typed AST."""
        typ = pnode.typ
        tok = pnode.tok
        children = pnode.children

        #if typ in self.number2symbol:  # non-terminal
        if ISNONTERMINAL(typ):
            c = '-' if not children else len(children)
            #log('non-terminal %s %s', nt_name, c)

            if typ == grammar_nt.lvalue_list:
                return self._AssocBinary(children)

            if typ == grammar_nt.atom:
                if children[0].tok.id == Id.Op_LParen:
                    return self.Expr(children[1])
                else:
                    raise NotImplementedError

            if typ == grammar_nt.eval_input:
                # testlist_input: testlist NEWLINE* ENDMARKER
                return self.Expr(children[0])

            if typ == grammar_nt.testlist:
                # testlist: test (',' test)* [',']
                return self._AssocBinary(children)

            elif typ == grammar_nt.arith_expr:
                # expr: term (('+'|'-') term)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.term:
                # term: factor (('*'|'/'|'div'|'mod') factor)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.expr:
                # expr: xor_expr ('|' xor_expr)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.shift_expr:
                # shift_expr: arith_expr (('<<'|'>>') arith_expr)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.comparison:
                # comparison: expr (comp_op expr)*
                return self._AssocBinary(children)

            elif typ == grammar_nt.factor:
                # factor: ('+'|'-'|'~') factor | power
                # the power would have already been reduced
                assert len(children) == 2, children
                op, e = children
                assert isinstance(op.tok, syntax_asdl.token)
                return expr.Unary(op.tok, self.Expr(e))

            elif typ == grammar_nt.atom_expr:
                # atom_expr: ['await'] atom trailer*

                # NOTE: This would be shorter in a recursive style.
                base = self.Expr(children[0])
                n = len(children)
                for i in xrange(1, n):
                    pnode = children[i]
                    tok = pnode.tok
                    base = self._Trailer(base, pnode)

                return base

            elif typ == grammar_nt.power:
                # power: atom_expr ['^' factor]

                # This doesn't repeat, so it doesn't matter if it's left or right
                # associative.
                return self._AssocBinary(children)

            elif typ == grammar_nt.array_literal:
                left_tok = children[0].tok

                # Approximation for now.
                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Lit_Chars
                ]
                array_words = [
                    word.CompoundWord([word_part.LiteralPart(t)])
                    for t in tokens
                ]  # type: List[word_t]
                return expr.ArrayLiteral(left_tok, array_words)

            elif typ == grammar_nt.regex_literal:
                left_tok = children[0].tok

                # Approximation for now.
                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Expr_Name
                ]
                parts = [regex.Var(t) for t in tokens]  # type: List[regex_t]

                return expr.RegexLiteral(left_tok, regex.Concat(parts))

            elif typ == grammar_nt.command_sub:
                left_tok = children[0].tok

                # Approximation for now.
                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Lit_Chars
                ]
                words = [
                    word.CompoundWord([word_part.LiteralPart(t)])
                    for t in tokens
                ]  # type: List[word_t]
                return expr.CommandSub(left_tok, command.SimpleCommand(words))

            elif typ == grammar_nt.expr_sub:
                left_tok = children[0].tok

                return expr.ExprSub(left_tok, self.Expr(children[1]))

            elif typ == grammar_nt.var_sub:
                left_tok = children[0].tok

                return expr.VarSub(left_tok, self.Expr(children[1]))

            elif typ == grammar_nt.dq_string:
                left_tok = children[0].tok

                tokens = [
                    pnode.tok for pnode in children[1:-1]
                    if pnode.tok.id == Id.Lit_Chars
                ]
                parts2 = [oil_word_part.Literal(t)
                          for t in tokens]  # type: List[oil_word_part_t]
                return expr.DoubleQuoted(left_tok, parts2)

            else:
                nt_name = self.number2symbol[typ]
                raise AssertionError("PNode type %d (%s) wasn't handled" %
                                     (typ, nt_name))

        else:  # Terminals should have a token
            #log('terminal %s', tok)

            if tok.id == Id.Expr_Name:
                return expr.Var(tok)
            elif tok.id == Id.Expr_Digits:
                return expr.Const(tok)

            else:
                raise AssertionError(tok.id)