def testTokens(self):
print(Id.Op_Newline)
print(ast.token(Id.Op_Newline, '\n'))
print(IdName(Id.Op_Newline))
print(Kind.Eof)
print(Kind.Left)
print('--')
for name in dir(Kind):
if name[0].isupper():
print(name, getattr(Kind, name))
# Make sure we're not exporting too much
print(dir(id_kind))
# 206 out of 256 tokens now
print(len(id_kind._ID_NAMES))
t = ast.token(Id.Arith_Plus, '+')
self.assertEqual(Kind.Arith, LookupKind(t.id))
t = ast.token(Id.Arith_CaretEqual, '^=')
self.assertEqual(Kind.Arith, LookupKind(t.id))
t = ast.token(Id.Arith_RBrace, '}')
self.assertEqual(Kind.Arith, LookupKind(t.id))
t = ast.token(Id.BoolBinary_DEqual, '==')
self.assertEqual(Kind.BoolBinary, LookupKind(t.id))
def _Read(self, lex_mode):
if self.line_lexer.AtEnd():
line_id, line = self.line_reader.GetLine()
if line is None: # no more lines
t = ast.token(Id.Eof_Real, '', -1)
# No line number. I guess we are showing the last line of the file.
# TODO: Could keep track of previous position for this case?
return t
self.line_lexer.Reset(line, line_id)
t = self.line_lexer.Read(lex_mode)
# e.g. translate ) or ` into EOF
if self.translation_stack:
old_id, new_id = self.translation_stack[-1] # top
if t.id == old_id:
new_s = IdName(new_id)
#print('==> TRANSLATING %s ==> %s' % (t, new_s))
self.translation_stack.pop()
#print(self.translation_stack)
t.id = new_id
return t
def Eat(self, token_type):
""" Eat()? """
if not self.AtToken(token_type):
t = IdName(token_type)
raise TdopParseError('TDOP expected %s, got %s' %
(t, self.cur_word))
self.Next()
def PushHint(self, old_id, new_id):
"""
Use cases:
Id.Op_RParen -> Id.Right_Subshell -- disambiguate
Id.Op_RParen -> Id.Eof_RParen
Problems for $() nesting.
- posix:
- case foo) and case (foo)
- func() {}
- subshell ( )
- bash extensions:
- precedence in [[, e.g. [[ (1 == 2) && (2 == 3) ]]
- arrays: a=(1 2 3), a+=(4 5)
"""
old_s = IdName(old_id)
new_s = IdName(new_id)
#print('* Lexer.PushHint %s => %s' % (old_s, new_s))
self.translation_stack.append((old_id, new_id))
def LeftAssign(p, w, left, rbp):
""" Normal binary operator like 1+2 or 2*3, etc. """
# x += 1, or a[i] += 1
if not IsLValue(left):
raise TdopParseError("Can't assign to %r (%s)" %
(left, IdName(left.id)))
# HACK: NullConstant makes this of type RightVar? Change that to something
# generic?
if left.tag == arith_expr_e.RightVar:
lhs = ast.LeftVar(left.name)
elif left.tag == arith_expr_e.ArithBinary:
assert left.op_id == Id.Arith_LBracket
# change a[i] to LeftIndex(a, i)
lhs = ast.LeftIndex(left.left, left.right)
else:
raise AssertionError
return ast.ArithAssign(word.ArithId(w), lhs, p.ParseUntil(rbp))
def Eval(self, node):
#print('!!', node.tag)
if node.tag == bool_expr_e.WordTest:
s = self._EvalCompoundWord(node.w)
return bool(s)
if node.tag == bool_expr_e.LogicalNot:
b = self.Eval(node.child)
return not b
if node.tag == bool_expr_e.LogicalAnd:
# Short-circuit evaluation
if self.Eval(node.left):
return self.Eval(node.right)
else:
return False
if node.tag == bool_expr_e.LogicalOr:
if self.Eval(node.left):
return True
else:
return self.Eval(node.right)
if node.tag == bool_expr_e.BoolUnary:
op_id = node.op_id
s = self._EvalCompoundWord(node.child)
# Now dispatch on arg type
arg_type = BOOL_OPS[op_id]
if arg_type == OperandType.Path:
try:
mode = os.stat(s).st_mode
except OSError as e: # Python 3: FileNotFoundError
# TODO: Signal extra debug information?
#self._AddErrorContext("Error from stat(%r): %s" % (s, e))
return False
if op_id == Id.BoolUnary_f:
return stat.S_ISREG(mode)
if arg_type == OperandType.Str:
if op_id == Id.BoolUnary_z:
return not bool(s)
if op_id == Id.BoolUnary_n:
return bool(s)
raise NotImplementedError(op_id)
raise NotImplementedError(arg_type)
#if node.id == Id.Node_BinaryExpr:
if node.tag == bool_expr_e.BoolBinary:
op_id = node.op_id
s1 = self._EvalCompoundWord(node.left)
# Whehter to glob escape
do_fnmatch = op_id in (Id.BoolBinary_Equal, Id.BoolBinary_DEqual,
Id.BoolBinary_NEqual)
s2 = self._EvalCompoundWord(node.right, do_fnmatch=do_fnmatch)
# Now dispatch on arg type
arg_type = BOOL_OPS[op_id]
if arg_type == OperandType.Path:
st1 = os.stat(s1)
st2 = os.stat(s2)
if op_id == Id.BoolBinary_nt:
return True # TODO: test newer than (mtime)
if arg_type == OperandType.Int:
# NOTE: We assume they are constants like [[ 3 -eq 3 ]].
# Bash also allows [[ 1+2 -eq 3 ]].
i1 = self._StringToIntegerOrError(s1)
i2 = self._StringToIntegerOrError(s2)
if op_id == Id.BoolBinary_eq:
return i1 == i2
if op_id == Id.BoolBinary_ne:
return i1 != i2
raise NotImplementedError(op_id)
if arg_type == OperandType.Str:
# TODO:
# - Compare arrays. (Although bash coerces them to string first)
if op_id in (Id.BoolBinary_Equal, Id.BoolBinary_DEqual):
#log('Comparing %s and %s', s2, s1)
return libc.fnmatch(s2, s1)
if op_id == Id.BoolBinary_NEqual:
return not libc.fnmatch(s2, s1)
if op_id == Id.BoolBinary_EqualTilde:
# NOTE: regex matching can't fail if compilation succeeds.
match = libc.regex_match(s2, s1)
# TODO: BASH_REMATCH or REGEX_MATCH
if match == 1:
self._SetRegexMatches('TODO')
is_match = True
elif match == 0:
is_match = False
elif match == -1:
raise AssertionError(
"Invalid regex %r: should have been caught at compile time"
% s2)
else:
raise AssertionError
return is_match
if op_id == Id.Redir_Less: # pun
return s1 < s2
if op_id == Id.Redir_Great: # pun
return s1 > s2
raise NotImplementedError(op_id)
# We could have govered all node IDs
raise AssertionError(IdName(node.id))
def _Eval(self, node):
#print('!!', node.tag)
if node.tag == bool_expr_e.WordTest:
s = self._EvalCompoundWord(node.w)
return bool(s)
if node.tag == bool_expr_e.LogicalNot:
b = self._Eval(node.child)
return not b
if node.tag == bool_expr_e.LogicalAnd:
# Short-circuit evaluation
if self._Eval(node.left):
return self._Eval(node.right)
else:
return False
if node.tag == bool_expr_e.LogicalOr:
if self._Eval(node.left):
return True
else:
return self._Eval(node.right)
if node.tag == bool_expr_e.BoolUnary:
op_id = node.op_id
s = self._EvalCompoundWord(node.child)
# Now dispatch on arg type
arg_type = BOOL_OPS[op_id]
if arg_type == OperandType.Path:
try:
mode = os.stat(s).st_mode
except FileNotFoundError as e:
# TODO: Signal extra debug information?
#self._AddErrorContext("Error from stat(%r): %s" % (s, e))
return False
if op_id == Id.BoolUnary_f:
return stat.S_ISREG(mode)
if arg_type == OperandType.Str:
if op_id == Id.BoolUnary_z:
return not bool(s)
if op_id == Id.BoolUnary_n:
return bool(s)
raise NotImplementedError(op_id)
raise NotImplementedError(arg_type)
#if node.id == Id.Node_BinaryExpr:
if node.tag == bool_expr_e.BoolBinary:
op_id = node.op_id
s1 = self._EvalCompoundWord(node.left)
# Whehter to glob escape
do_glob = op_id in (Id.BoolBinary_Equal, Id.BoolBinary_DEqual,
Id.BoolBinary_NEqual)
s2 = self._EvalCompoundWord(node.right, do_glob=do_glob)
# Now dispatch on arg type
arg_type = BOOL_OPS[op_id]
if arg_type == OperandType.Path:
st1 = os.stat(s1)
st2 = os.stat(s2)
if op_id == Id.BoolBinary_nt:
return True # TODO: test newer than (mtime)
if arg_type == OperandType.Int:
try:
i1 = int(s1)
i2 = int(s2)
except ValueError as e:
# NOTE: Bash turns these into zero, but we won't by default. Could
# provide a compat option.
# Also I think this should turn into exit code 3:
# - 0 true / 1 false / 3 runtime error
# - 2 is for PARSE error.
raise ExprEvalError("Invalid integer: %s" % e)
if op_id == Id.BoolBinary_eq:
return i1 == i2
if op_id == Id.BoolBinary_ne:
return i1 != i2
raise NotImplementedError(op_id)
if arg_type == OperandType.Str:
# TODO:
# - Compare arrays. (Although bash coerces them to string first)
if op_id in (Id.BoolBinary_Equal, Id.BoolBinary_DEqual):
#return True, _ValuesAreEqual(val1, val2)
return libc.fnmatch(s2, s1)
if op_id == Id.BoolBinary_NEqual:
#return True, not _ValuesAreEqual(val1, val2)
return not libc.fnmatch(s2, s1)
if op_id == Id.BoolBinary_EqualTilde:
# NOTE: regex matching can't fail if compilation succeeds.
match = libc.regex_match(s2, s1)
# TODO: BASH_REMATCH or REGEX_MATCH
if match == 1:
self._SetRegexMatches('TODO')
is_match = True
elif match == 0:
is_match = False
elif match == -1:
raise AssertionError(
"Invalid regex %r: should have been caught at compile time"
% s2)
else:
raise AssertionError
return is_match
if op_id == Id.Redir_Less: # pun
return s1 < s2
if op_id == Id.Redir_Great: # pun
return s1 > s2
raise NotImplementedError(op_id)
# We could have govered all node IDs
raise AssertionError(IdName(node.id))