def _MatchOshToken_Fast(lex_mode, line, start_pos): # type: (lex_mode_t, str, int) -> Tuple[Id_t, int] """Returns (Id, end_pos).""" tok_type, end_pos = fastlex.MatchOshToken(lex_mode.enum_id, line, start_pos) # IMPORTANT: We're reusing Id instances here. Ids are very common, so this # saves memory. return IdInstance(tok_type), end_pos
def _ThreeArgs(w_parser):
"""Returns an expression tree to be evaluated."""
w0 = w_parser.Read()
w1 = w_parser.Read()
w2 = w_parser.Read()
# NOTE: Order is important here.
binary_id = _BINARY_LOOKUP.get(w1.s)
if binary_id is not None:
return bool_expr.BoolBinary(IdInstance(binary_id), w0, w2)
if w1.s == '-a':
return bool_expr.LogicalAnd(bool_expr.WordTest(w0),
bool_expr.WordTest(w2))
if w1.s == '-o':
return bool_expr.LogicalOr(bool_expr.WordTest(w0),
bool_expr.WordTest(w2))
if w0.s == '!':
w_parser.Rewind(2)
child = _TwoArgs(w_parser)
return bool_expr.LogicalNot(child)
if w0.s == '(' and w2.s == ')':
return bool_expr.WordTest(w1)
p_die('Expected binary operator, got %r (3 args)', w1.s, word=w1)
def _ThreeArgs(argv):
"""Returns an expression tree to be evaluated."""
a0, a1, a2 = argv
# NOTE: Order is important here.
binary_id = _BINARY_LOOKUP.get(a1)
if binary_id is not None:
left = word.StringWord(Id.Word_Compound, a0)
right = word.StringWord(Id.Word_Compound, a2)
return bool_expr.BoolBinary(IdInstance(binary_id), left, right)
if a1 == '-a':
left = _StringWordTest(a0)
right = _StringWordTest(a2)
return bool_expr.LogicalAnd(left, right)
if a1 == '-o':
left = _StringWordTest(a0)
right = _StringWordTest(a2)
return bool_expr.LogicalOr(left, right)
if a0 == '!':
child = _TwoArgs(argv[1:])
return bool_expr.LogicalNot(child)
if a0 == '(' and a2 == ')':
return _StringWordTest(a1)
p_die('Syntax error: binary operator expected, got %r (3 args)', a1)
def ReadWord(self, unused_lex_mode):
"""Interface for bool_parse.py."""
if self.i == self.n:
# Does it make sense to define Eof_Argv or something?
w = word.StringWord(Id.Eof_Real, '')
# TODO: Add a way to show this. Show 1 char past the right-most spid of
# the last word? But we only have the left-most spid.
w.spids.append(const.NO_INTEGER)
return w
#log('ARGV %s i %d', self.argv, self.i)
s = self.arg_vec.strs[self.i]
left_spid = self.arg_vec.spids[self.i]
self.i += 1
# default is an operand word
id_int = (_UNARY_LOOKUP.get(s) or _BINARY_LOOKUP.get(s)
or _OTHER_LOOKUP.get(s))
id_ = Id.Word_Compound if id_int is None else IdInstance(id_int)
# NOTE: We only have the left spid now. It might be useful to add the
# right one.
w = word.StringWord(id_, s)
w.spids.append(left_spid)
return w
def _TwoArgs(w_parser):
"""Returns an expression tree to be evaluated."""
w0 = w_parser.Read()
w1 = w_parser.Read()
if w0.s == '!':
return bool_expr.LogicalNot(bool_expr.WordTest(w1))
unary_id = _UNARY_LOOKUP.get(w0.s)
if unary_id is None:
# TODO:
# - separate lookup by unary
p_die('Expected unary operator, got %r (2 args)', w0.s, word=w0)
return bool_expr.BoolUnary(IdInstance(unary_id), w1)
def _TwoArgs(argv):
"""Returns an expression tree to be evaluated."""
a0, a1 = argv
if a0 == '!':
return bool_expr.LogicalNot(_StringWordTest(a1))
unary_id = _UNARY_LOOKUP.get(a0)
if unary_id is None:
# TODO:
# - syntax error
# - separate lookup by unary
p_die('Expected unary operator, got %r (2 args)', a0)
child = word.StringWord(Id.Word_Compound, a1)
return bool_expr.BoolUnary(IdInstance(unary_id), child)
def ReadWord(self, unused_lex_mode):
if self.i == self.n:
# NOTE: Could define something special
return word.StringWord(Id.Eof_Real, '')
#log('ARGV %s i %d', self.argv, self.i)
s = self.argv[self.i]
self.i += 1
# default is an operand word
id_int = (_UNARY_LOOKUP.get(s) or _BINARY_LOOKUP.get(s)
or _OTHER_LOOKUP.get(s))
id_ = Id.Word_Compound if id_int is None else IdInstance(id_int)
return word.StringWord(id_, s)
def _MatchHistoryToken_Fast(line, start_pos): # type: (str, int) -> Tuple[Id_t, int] """Returns (id, end_pos).""" tok_type, end_pos = fastlex.MatchHistoryToken(line, start_pos) return IdInstance(tok_type), end_pos
def PrintBoolTable():
for i, arg_type in BOOL_ARG_TYPES.items():
row = (IdInstance(i), arg_type)
print('\t'.join(str(c) for c in row))
def testEquality(self):
left = IdInstance(198)
right = IdInstance(198)
print(left, right)
print(left == right)
self.assertEqual(left, right)
def _MatchHistoryToken_Fast(line, start_pos):
"""Returns (id, end_pos)."""
tok_type, end_pos = fastlex.MatchHistoryToken(line, start_pos)
return IdInstance(tok_type), end_pos
def PrintBoolTable():
for i, arg_type in BOOL_ARG_TYPES.items():
print('%-40s %s' % (IdInstance(i), arg_type))
def Expr(self, pnode):
# type: (PNode) -> expr_t
"""Transform expressions (as opposed to statements)."""
typ = pnode.typ
tok = pnode.tok
children = pnode.children
if ISNONTERMINAL(typ):
#
# Oil Entry Points / Additions
#
if typ == grammar_nt.oil_expr: # for if/while
# oil_expr: '(' testlist ')'
return self.Expr(children[1])
if typ == grammar_nt.return_expr:
# return_expr: testlist end_stmt
return self.Expr(children[0])
if typ == grammar_nt.place_list:
return self._AssocBinary(children)
if typ == grammar_nt.place:
# place: NAME place_trailer*
if len(pnode.children) == 1:
return self.Expr(pnode.children[0])
# TODO: Called _Trailer but don't handle ( )?
# only [] . -> :: ?
raise NotImplementedError
#
# Python-like Expressions / Operators
#
if typ == grammar_nt.atom:
if len(children) == 1:
return self.Expr(children[0])
return self._Atom(children)
if typ == grammar_nt.testlist:
# testlist: test (',' test)* [',']
# We need tuples for Python's 'var a, b = x' and 'for (a, b in x) {'
return self._Tuple(children)
if typ == grammar_nt.test:
# test: or_test ['if' or_test 'else' test] | lambdef
if len(children) == 1:
return self.Expr(children[0])
# TODO: Handle lambdef
test = self.Expr(children[2])
body = self.Expr(children[0])
orelse = self.Expr(children[4])
return expr.IfExp(test, body, orelse)
if typ == grammar_nt.test_nocond:
# test_nocond: or_test | lambdef_nocond
assert len(children) == 1
return self.Expr(children[0])
if typ == grammar_nt.argument:
# argument: ( test [comp_for] |
# test '=' test |
# '**' test |
# '*' test )
if len(pnode.children) == 1:
return self.Expr(children[0])
# TODO:
raise NotImplementedError
if typ == grammar_nt.subscript:
# subscript: test | [test] ':' [test] [sliceop]
if len(pnode.children) == 1:
return self.Expr(children[0])
# TODO:
raise NotImplementedError
if typ == grammar_nt.testlist_comp:
# testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
if children[1].typ == grammar_nt.comp_for:
elt = self.Expr(children[0])
comp = self._CompFor(children[1])
return expr.ListComp(elt, [comp])
# (1,) (1, 2) etc.
if children[1].tok.id == Id.Arith_Comma:
return self._Tuple(children)
raise NotImplementedError('testlist_comp')
elif typ == grammar_nt.exprlist:
# exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
if len(children) == 1:
return self.Expr(children[0])
# used in for loop, genexpr.
# TODO: This sould be placelist? for x, *y ?
raise NotImplementedError('exprlist')
#
# Operators with Precedence
#
if typ == grammar_nt.or_test:
# or_test: and_test ('or' and_test)*
return self._AssocBinary(children)
if typ == grammar_nt.and_test:
# and_test: not_test ('and' not_test)*
return self._AssocBinary(children)
if typ == grammar_nt.not_test:
# not_test: 'not' not_test | comparison
if len(children) == 1:
return self.Expr(children[0])
op_tok = children[0].tok # not
return expr.Unary(op_tok, self.Expr(children[1]))
elif typ == grammar_nt.comparison:
if len(children) == 1:
return self.Expr(children[0])
return self._CompareChain(children)
elif typ == grammar_nt.expr:
# expr: xor_expr ('|' xor_expr)*
return self._AssocBinary(children)
if typ == grammar_nt.xor_expr:
# xor_expr: and_expr ('xor' and_expr)*
return self._AssocBinary(children)
if typ == grammar_nt.and_expr: # a & b
# and_expr: shift_expr ('&' shift_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.shift_expr:
# shift_expr: arith_expr (('<<'|'>>') arith_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.arith_expr:
# arith_expr: term (('+'|'-') term)*
return self._AssocBinary(children)
elif typ == grammar_nt.term:
# term: factor (('*'|'/'|'div'|'mod') factor)*
return self._AssocBinary(children)
elif typ == grammar_nt.factor:
# factor: ('+'|'-'|'~') factor | power
# the power would have already been reduced
if len(children) == 1:
return self.Expr(children[0])
op, e = children
assert isinstance(op.tok, token)
return expr.Unary(op.tok, self.Expr(e))
elif typ == grammar_nt.power:
# power: atom trailer* ['^' factor]
node = self.Expr(children[0])
if len(children) == 1: # No trailers
return node
n = len(children)
i = 1
while i < n and ISNONTERMINAL(children[i].typ):
node = self._Trailer(node, children[i])
i += 1
if i != n: # ['^' factor]
op_tok = children[i].tok
assert op_tok.id == Id.Arith_Caret, op_tok
factor = self.Expr(children[i+1])
node = expr.Binary(op_tok, node, factor)
return node
#
# Oil Lexer Modes
#
elif typ == grammar_nt.array_literal:
left_tok = children[0].tok
# Approximation for now.
tokens = [
pnode.tok for pnode in children[1:-1] if pnode.tok.id ==
Id.Lit_Chars
]
items = [expr.Const(t) for t in tokens] # type: List[expr_t]
return expr.ArrayLiteral(left_tok, items)
elif typ == grammar_nt.sh_array_literal:
left_tok = children[0].tok
# HACK: When typ is Id.Expr_CastedDummy, the 'tok' field ('opaque')
# actually has a list of words!
typ1 = children[1].typ
assert typ1 == Id.Expr_CastedDummy.enum_id, typ1
array_words = cast('List[word_t]', children[1].tok)
return sh_array_literal(left_tok, array_words)
elif typ == grammar_nt.sh_command_sub:
return cast(command_sub, children[1].tok)
elif typ == grammar_nt.braced_var_sub:
return cast(braced_var_sub, children[1].tok)
elif typ == grammar_nt.dq_string:
return cast(double_quoted, children[1].tok)
elif typ == grammar_nt.sq_string:
return cast(single_quoted, children[1].tok)
elif typ == grammar_nt.simple_var_sub:
return simple_var_sub(children[0].tok)
else:
nt_name = self.number2symbol[typ]
raise AssertionError(
"PNode type %d (%s) wasn't handled" % (typ, nt_name))
else: # Terminals should have a token
id_ = tok.id
if id_ == Id.Expr_Name:
return expr.Var(tok)
if id_ in (
Id.Expr_DecInt, Id.Expr_BinInt, Id.Expr_OctInt, Id.Expr_HexInt,
Id.Expr_Float):
return expr.Const(tok)
if id_ in (Id.Expr_Null, Id.Expr_True, Id.Expr_False):
return expr.Const(tok)
from core.meta import IdInstance
raise NotImplementedError(IdInstance(typ))
def MatchOshToken(lex_mode, line, start_pos):
tok_type, end_pos = fastlex.MatchOshToken(lex_mode.enum_id, line,
start_pos)
return IdInstance(tok_type), end_pos
def Expr(self, pnode):
# type: (PNode) -> expr_t
"""Transform expressions (as opposed to statements)."""
typ = pnode.typ
tok = pnode.tok
children = pnode.children
if ISNONTERMINAL(typ):
#
# Oil Entry Points / Additions
#
if typ == grammar_nt.oil_expr: # for if/while
# oil_expr: '(' testlist ')'
return self.Expr(children[1])
if typ == grammar_nt.command_expr:
# return_expr: testlist end_stmt
return self.Expr(children[0])
#
# Python-like Expressions / Operators
#
if typ == grammar_nt.atom:
if len(children) == 1:
return self.Expr(children[0])
return self._Atom(children)
if typ == grammar_nt.testlist:
# testlist: test (',' test)* [',']
return self._Tuple(children)
if typ == grammar_nt.test:
# test: or_test ['if' or_test 'else' test] | lambdef
if len(children) == 1:
return self.Expr(children[0])
# TODO: Handle lambdef
test = self.Expr(children[2])
body = self.Expr(children[0])
orelse = self.Expr(children[4])
return expr.IfExp(test, body, orelse)
if typ == grammar_nt.lambdef:
# lambdef: '|' [name_type_list] '|' test
n = len(children)
if n == 4:
params = self._NameTypeList(children[1])
else:
params = []
body = self.Expr(children[n - 1])
return expr.Lambda(params, body)
#
# Operators with Precedence
#
if typ == grammar_nt.or_test:
# or_test: and_test ('or' and_test)*
return self._AssocBinary(children)
if typ == grammar_nt.and_test:
# and_test: not_test ('and' not_test)*
return self._AssocBinary(children)
if typ == grammar_nt.not_test:
# not_test: 'not' not_test | comparison
if len(children) == 1:
return self.Expr(children[0])
op_tok = children[0].tok # not
return expr.Unary(op_tok, self.Expr(children[1]))
elif typ == grammar_nt.comparison:
if len(children) == 1:
return self.Expr(children[0])
return self._CompareChain(children)
elif typ == grammar_nt.range_expr:
if len(children) == 1:
return self.Expr(children[0])
if len(children) == 3:
return expr.Range(self.Expr(children[0]),
self.Expr(children[2]))
raise AssertionError(children)
elif typ == grammar_nt.expr:
# expr: xor_expr ('|' xor_expr)*
return self._AssocBinary(children)
if typ == grammar_nt.xor_expr:
# xor_expr: and_expr ('xor' and_expr)*
return self._AssocBinary(children)
if typ == grammar_nt.and_expr: # a & b
# and_expr: shift_expr ('&' shift_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.shift_expr:
# shift_expr: arith_expr (('<<'|'>>') arith_expr)*
return self._AssocBinary(children)
elif typ == grammar_nt.arith_expr:
# arith_expr: term (('+'|'-') term)*
return self._AssocBinary(children)
elif typ == grammar_nt.term:
# term: factor (('*'|'/'|'div'|'mod') factor)*
return self._AssocBinary(children)
elif typ == grammar_nt.factor:
# factor: ('+'|'-'|'~') factor | power
# the power would have already been reduced
if len(children) == 1:
return self.Expr(children[0])
op, e = children
assert isinstance(op.tok, token)
return expr.Unary(op.tok, self.Expr(e))
elif typ == grammar_nt.power:
# power: atom trailer* ['^' factor]
node = self.Expr(children[0])
if len(children) == 1: # No trailers
return node
n = len(children)
i = 1
while i < n and ISNONTERMINAL(children[i].typ):
node = self._Trailer(node, children[i])
i += 1
if i != n: # ['^' factor]
op_tok = children[i].tok
assert op_tok.id == Id.Arith_Caret, op_tok
factor = self.Expr(children[i + 1])
node = expr.Binary(op_tok, node, factor)
return node
elif typ == grammar_nt.array_literal:
left_tok = children[0].tok
items = [self._ArrayItem(p) for p in children[1:-1]]
return expr.ArrayLiteral(left_tok, items)
elif typ == grammar_nt.oil_expr_sub:
return self.Expr(children[0])
#
# Oil Lexer Modes
#
elif typ == grammar_nt.sh_array_literal:
left_tok = children[0].tok
# HACK: When typ is Id.Expr_CastedDummy, the 'tok' field ('opaque')
# actually has a list of words!
typ1 = children[1].typ
assert typ1 == Id.Expr_CastedDummy.enum_id, typ1
array_words = cast('List[word_t]', children[1].tok)
return sh_array_literal(left_tok, array_words)
elif typ == grammar_nt.sh_command_sub:
return cast(command_sub, children[1].tok)
elif typ == grammar_nt.braced_var_sub:
return cast(braced_var_sub, children[1].tok)
elif typ == grammar_nt.dq_string:
return cast(double_quoted, children[1].tok)
elif typ == grammar_nt.sq_string:
return cast(single_quoted, children[1].tok)
elif typ == grammar_nt.simple_var_sub:
return simple_var_sub(children[0].tok)
else:
nt_name = self.number2symbol[typ]
raise AssertionError("PNode type %d (%s) wasn't handled" %
(typ, nt_name))
else: # Terminals should have a token
id_ = tok.id
if id_ == Id.Expr_Name:
return expr.Var(tok)
if id_ in (Id.Expr_DecInt, Id.Expr_BinInt, Id.Expr_OctInt,
Id.Expr_HexInt, Id.Expr_Float):
return expr.Const(tok)
if id_ in (Id.Expr_Null, Id.Expr_True, Id.Expr_False):
return expr.Const(tok)
from core.meta import IdInstance
raise NotImplementedError(IdInstance(typ))
def MatchOshToken(lex_mode, line, start_pos):
tok_type, end_pos = fastlex.MatchOshToken(lex_mode.enum_id, line, start_pos)
#log('tok_type = %d, id = %s', tok_type, IdInstance(tok_type))
return IdInstance(tok_type), end_pos