def Run(self, cmd_val):
# type: (cmd_value__Argv) -> int
attrs, arg_r = flag_spec.ParseCmdVal('read', cmd_val)
arg = arg_types.read(attrs.attrs)
names = arg_r.Rest()
fd = self.stdin.fileno()
if arg.t >= 0.0:
if arg.t != 0.0:
e_die("read -t isn't implemented (except t=0)")
else:
return 0 if pyos.InputAvailable(fd) else 1
bits = 0
if self.stdin.isatty():
bits |= pyos.TERM_ICANON
if arg.s: # silent
bits |= pyos.TERM_ECHO
if arg.p is not None: # only if tty
mylib.Stderr().write(arg.p)
if bits == 0:
status = self._Read(arg, names)
else:
term = pyos.TermState(fd, ~bits)
try:
status = self._Read(arg, names)
finally:
term.Restore()
return status
def EvalArithLhs(self, anode, span_id):
# type: (arith_expr_t, int) -> lvalue_t
"""
For (( a[x] = 1 )) etc.
"""
UP_anode = anode
if anode.tag_() == arith_expr_e.Binary:
anode = cast(arith_expr__Binary, UP_anode)
if anode.op_id == Id.Arith_LBracket:
var_name, span_id = self._VarRefOrWord(anode.left)
if var_name is not None:
if self.mem.IsAssocArray(var_name, scope_e.Dynamic):
key = self.EvalWordToString(anode.right)
lval2 = lvalue.Keyed(var_name, key)
lval2.spids.append(span_id)
lval = lval2 # type: lvalue_t
return lval
else:
index = self.EvalToInt(anode.right)
lval3 = lvalue.Indexed(var_name, index)
lval3.spids.append(span_id)
lval = lval3
return lval
var_name, span_id = self._VarRefOrWord(anode)
if var_name is not None:
lval1 = lvalue.Named(var_name)
lval1.spids.append(span_id)
lval = lval1
return lval
# e.g. unset 'x-y'. status 2 for runtime parse error
e_die('Invalid place to modify', span_id=span_id, status=2)
def _MakeProcess(self, node, inherit_errexit=True):
# type: (command_t, bool) -> process.Process
"""
Assume we will run the node in another process. Return a process.
"""
UP_node = node
if node.tag_() == command_e.ControlFlow:
node = cast(command__ControlFlow, UP_node)
# Pipeline or subshells with control flow are invalid, e.g.:
# - break | less
# - continue | less
# - ( return )
# NOTE: This could be done at parse time too.
if node.token.id != Id.ControlFlow_Exit:
e_die('Invalid control flow %r in pipeline / subshell / background',
node.token.val, token=node.token)
# NOTE: If ErrExit(), we could be verbose about subprogram errors? This
# only really matters when executing 'exit 42', because the child shell
# inherits errexit and will be verbose. Other notes:
#
# - We might want errors to fit on a single line so they don't get
# interleaved.
# - We could turn the `exit` builtin into a FatalRuntimeError exception and
# get this check for "free".
thunk = process.SubProgramThunk(self.cmd_ev, node,
inherit_errexit=inherit_errexit)
p = process.Process(thunk, self.job_state)
return p
def Replace(self, s, op):
# type: (str, suffix_op__PatSub) -> str
regex = '(%s)' % self.regex # make it a group
if op.replace_mode == Id.Lit_Slash:
try:
return _PatSubAll(s, regex,
self.replace_str) # loop over matches
except RuntimeError as e:
# libc.regex_first_group_match raises RuntimeError.
# note: MyPy doesn't know RuntimeError has e.message (and e.args)
msg = e.message # type: str
e_die('Error matching regex %r: %s',
regex,
msg,
span_id=self.slash_spid)
if op.replace_mode == Id.Lit_Pound:
regex = '^' + regex
elif op.replace_mode == Id.Lit_Percent:
regex = regex + '$'
m = libc.regex_first_group_match(regex, s, 0)
#log('regex = %r, s = %r, match = %r', regex, s, m)
if m is None:
return s
start, end = m
return s[:start] + self.replace_str + s[end:]
def isatty(fd, s, blame_word):
# type: (int, str, word_t) -> bool
try:
return posix.isatty(fd)
# fd is user input, and causes this exception in the binding.
except OverflowError:
e_die('File descriptor %r is too big', s, word=blame_word)
def EvalWordToString(self, node):
# type: (arith_expr_t) -> str
"""
Args:
node: arith_expr_t
Returns:
str
Raises:
error.FatalRuntime if the expression isn't a string
Or if it contains a bare variable like a[x]
These are allowed because they're unambiguous, unlike a[x]
a[$x] a["$x"] a["x"] a['x']
"""
UP_node = node
if node.tag_() == arith_expr_e.Word: # $(( $x )) $(( ${x}${y} )), etc.
w = cast(compound_word, UP_node)
val = self.word_ev.EvalWordToString(w)
return val.s
else:
# TODO: location info for orginal
e_die("Associative array keys must be strings: $x 'x' \"$x\" etc.")
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(
cmd_val.arg_spids) else runtime.NO_SPID
arg0 = argv[0]
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# Copied from core/executor.py
if call_procs:
proc_node = self.procs.get(arg0)
if proc_node is not None:
if (self.exec_opts.strict_errexit()
and self.mutable_opts.ErrExitIsDisabled()):
# TODO: make errfmt a member
#self.errfmt.Print_('errexit was disabled for this construct',
# span_id=self.mutable_opts.errexit.spid_stack[0])
#stderr_line('')
e_die(
"Can't run a proc while errexit is disabled. "
"Use 'catch' or wrap it in a process with $0 myproc",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(proc_node, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# See how many tests will pass
#if mylib.PYTHON:
if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling
import subprocess
try:
status = subprocess.call(cmd_val.argv)
except OSError as e:
log('Error running %s: %s', cmd_val.argv, e)
return 1
return status
log('Unhandled SimpleCommand')
f = mylib.Stdout()
#ast_f = fmt.DetectConsoleOutput(f)
# Stupid Eclipse debugger doesn't display ANSI
ast_f = fmt.TextOutput(f)
tree = cmd_val.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
return 0
def _LookupVar(name, mem, exec_opts):
# type: (str, Mem, optview.Exec) -> value_t
val = mem.GetVar(name)
# By default, undefined variables are the ZERO value. TODO: Respect
# nounset and raise an exception.
if val.tag_() == value_e.Undef and exec_opts.nounset():
e_die('Undefined variable %r', name) # TODO: need token
return val
def EvalPlusEquals(self, lval, rhs_py):
# type: (lvalue_t, Union[int, float]) -> Union[int, float]
lhs_py = self.LookupVar(lval.name)
if not isinstance(lhs_py, (int, float)):
# TODO: Could point at the variable name
e_die("Object of type %r doesn't support +=", lhs_py.__class__.__name__)
return lhs_py + rhs_py
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(
cmd_val.arg_spids) else runtime.NO_SPID
arg0 = argv[0]
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
func_node = self.procs.get(arg0)
if func_node is not None:
if (self.exec_opts.strict_errexit()
and self.mutable_opts.errexit.SpidIfDisabled() !=
runtime.NO_SPID):
# NOTE: This would be checked below, but this gives a better error
# message.
e_die(
"can't disable errexit running a function. "
"Maybe wrap the function in a process with the at-splice "
"pattern.",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(func_node, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# See how many tests will pass
#if mylib.PYTHON:
if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling
import subprocess
try:
status = subprocess.call(cmd_val.argv)
except OSError as e:
log('Error running %s: %s', cmd_val.argv, e)
return 1
return status
log('Unhandled SimpleCommand')
f = mylib.Stdout()
#ast_f = fmt.DetectConsoleOutput(f)
# Stupid Eclipse debugger doesn't display ANSI
ast_f = fmt.TextOutput(f)
tree = cmd_val.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
return 0
def Run(self, cmd_val):
# type: (cmd_value__Argv) -> int
# TODO: Also hard usage error here too?
attrs, arg_r = flag_spec.ParseOilCmdVal('run', cmd_val)
arg = arg_types.run(attrs.attrs)
if arg_r.Peek() is None:
# HARD ERROR, not e_usage(), because errexit is often disabled!
e_die("'run' expected a command to run", status=2)
argv, spids = arg_r.Rest2()
cmd_val2 = cmd_value.Argv(argv, spids, cmd_val.block)
# Set in the 'except' block, e.g. if 'myfunc' failed
failure_spid = runtime.NO_SPID
try:
# Temporarily turn ON errexit, and blame the 'run' spid. Note that
# 'if run myproc' disables it and then enables it!
with state.ctx_ErrExit(self.mutable_opts, True,
cmd_val.arg_spids[0]):
# Pass do_fork=True. Slight annoyance: the real value is a field of
# command.Simple(). See _NoForkLast() in CommandEvaluator We have an
# extra fork (miss out on an optimization) of code like ( status ls )
# or forkwait { status ls }, but that is NOT idiomatic code. status is
# for functions.
status = self.shell_ex.RunSimpleCommand(cmd_val2, True)
#log('st %d', status)
except error.ErrExit as e: # from functino call
#log('e %d', e.exit_status)
status = e.exit_status
failure_spid = e.span_id
# Do this before -allow-status-01
if arg.status_ok is not None:
status = _AdjustStatus(arg.status_ok, status)
if arg.allow_status_01 and status not in (0, 1):
if failure_spid != runtime.NO_SPID:
self.errfmt.Print_('(original failure)', span_id=failure_spid)
self.errfmt.StderrLine('')
raise error.ErrExit('fatal: status %d when --allow-status-01' %
status,
span_id=spids[0],
status=status)
if arg.assign_status is not None:
var_name = arg.assign_status
if var_name.startswith(':'):
var_name = var_name[1:]
state.SetRefString(self.mem, var_name, str(status))
return 0 # don't fail
return status
def isatty(fd_str, blame_word):
# type: (str, word_t) -> bool
try:
fd = int(fd_str)
except ValueError:
e_die('Invalid file descriptor %r', fd_str, word=blame_word)
try:
return posix.isatty(fd)
# fd is user input, and causes this exception in the binding.
except OverflowError:
e_die('File descriptor %r is too big', fd_str, word=blame_word)
def _UnsetVar(self, arg, spid, proc_fallback):
# type: (str, int, bool) -> bool
"""
Returns:
bool: whether the 'unset' builtin should succeed with code 0.
"""
arena = self.parse_ctx.arena
a_parser = self.parse_ctx.MakeArithParser(arg)
arena.PushSource(source.ArgvWord(spid))
try:
anode = a_parser.Parse()
except error.Parse as e:
# show parse error
ui.PrettyPrintError(e, arena)
# point to word
e_usage('Invalid unset expression', span_id=spid)
finally:
arena.PopSource()
lval = self.arith_ev.EvalArithLhs(anode, spid)
# Prevent attacks like these by default:
#
# unset -v 'A["$(echo K; rm *)"]'
if not self.exec_opts.eval_unsafe_arith(
) and lval.tag_() != lvalue_e.Named:
e_die(
'Expected a variable name. Expressions are allowed with shopt -s eval_unsafe_arith',
span_id=spid)
#log('lval %s', lval)
found = False
try:
# not strict
found = self.mem.Unset(lval, scope_e.Dynamic, False)
except error.Runtime as e:
# note: in bash, myreadonly=X fails, but declare myreadonly=X doens't
# fail because it's a builtin. So I guess the same is true of 'unset'.
e.span_id = spid
ui.PrettyPrintError(e, arena)
return False
if proc_fallback and not found:
if arg in self.funcs:
del self.funcs[arg]
return True
def LookupVar(self, var_name):
"""Convert to a Python object so we can calculate on it natively."""
# Lookup WITHOUT dynamic scope.
val = self.mem.GetValue(var_name, which_scopes=scope_e.LocalOrGlobal)
if val.tag == value_e.Undef:
# TODO: Location info
e_die('Undefined variable %r', var_name)
if val.tag == value_e.Str:
return val.s
if val.tag == value_e.MaybeStrArray:
return val.strs # node: has None
if val.tag == value_e.AssocArray:
return val.d
if val.tag == value_e.Obj:
return val.obj
def _ValToIntOrError(self, val, span_id=runtime.NO_SPID):
# type: (value_t, int) -> int
try:
UP_val = val
with tagswitch(val) as case:
if case(value_e.Undef
): # 'nounset' already handled before got here
# Happens upon a[undefined]=42, which unfortunately turns into a[0]=42.
#log('blame_word %s arena %s', blame_word, self.arena)
e_strict('Undefined value in arithmetic context',
span_id=span_id)
elif case(value_e.Int):
val = cast(value__Int, UP_val)
return val.i
elif case(value_e.Str):
val = cast(value__Str, UP_val)
return self._StringToInteger(
val.s, span_id=span_id) # calls e_strict
elif case(value_e.Obj):
# Note: this handles var x = 42; echo $(( x > 2 )).
if mylib.PYTHON:
val = cast(value__Obj, UP_val)
if isinstance(val.obj, int):
return val.obj
raise AssertionError() # not in C++
except error.Strict as e:
if self.exec_opts.strict_arith():
raise
else:
return 0
# Arrays and associative arrays always fail -- not controlled by
# strict_arith.
# In bash, (( a )) is like (( a[0] )), but I don't want that.
# And returning '0' gives different results.
e_die("Expected a value convertible to integer, got %s",
ui.ValType(val),
span_id=span_id)
def _ClassLiteralToPosixEre(term, parts):
# type: (class_literal_term_t, List[str]) -> None
UP_term = term
tag = term.tag_()
if tag == class_literal_term_e.Range:
term = cast(class_literal_term__Range, UP_term)
# \\ \^ \- can be used in ranges?
start = glob_.EreCharClassEscape(term.start)
end = glob_.EreCharClassEscape(term.end)
parts.append('%s-%s' % (start, end))
return
if tag == class_literal_term_e.ByteSet:
term = cast(class_literal_term__ByteSet, UP_term)
# This escaping is different than ExtendedRegexEscape.
parts.append(glob_.EreCharClassEscape(term.bytes))
return
if tag == class_literal_term_e.CodePoint:
term = cast(class_literal_term__CodePoint, UP_term)
code_point = term.i
if code_point < 128:
parts.append(chr(code_point))
else:
e_die("ERE can't express code point %d",
code_point,
span_id=term.spid)
return
if tag == class_literal_term_e.PerlClass:
term = cast(perl_class, UP_term)
n = term.name
chars = PERL_CLASS[term.name] # looks like '[:digit:]'
if term.negated:
e_die("Perl classes can't be negated in ERE",
span_id=term.negated.span_id)
else:
pat = '%s' % chars
parts.append(pat)
return
if tag == class_literal_term_e.PosixClass:
term = cast(posix_class, UP_term)
n = term.name # looks like 'digit'
if term.negated:
e_die("POSIX classes can't be negated in ERE",
span_id=term.negated.span_id)
else:
pat = '[:%s:]' % n
parts.append(pat)
return
if tag == class_literal_term_e.CharLiteral:
term = cast(class_literal_term__CharLiteral, UP_term)
parts.append(term.tok.val)
return
raise NotImplementedError(tag)
def _MutateClassLiteral(self, node):
# type: (re_t) -> None
for i, term in enumerate(node.terms):
s = None
if term.tag == class_literal_term_e.SingleQuoted:
s = word_eval.EvalSingleQuoted(term)
spid = term.left.span_id
elif term.tag == class_literal_term_e.DoubleQuoted:
s = self.word_ev.EvalDoubleQuotedToString(term)
spid = term.left.span_id
elif term.tag == class_literal_term_e.BracedVarSub:
s = self.word_ev.EvalBracedVarSubToString(term)
spid = term.spids[0]
elif term.tag == class_literal_term_e.SimpleVarSub:
s = self.word_ev.EvalSimpleVarSubToString(term.token)
spid = term.token.span_id
elif term.tag == class_literal_term_e.CharLiteral:
# What about \0?
# At runtime, ERE should disallow it. But we can also disallow it here.
new_leaf = word_compile.EvalCharLiteralForRegex(term.tok)
if new_leaf:
node.terms[i] = new_leaf
if s is not None:
# A string like '\x7f\xff' should be presented like
if len(s) > 1:
for c in s:
if ord(c) > 128:
e_die(
"Express these bytes as character literals to avoid "
"confusing them with encoded characters",
span_id=spid)
node.terms[i] = class_literal_term.ByteSet(s, spid)
def _Maybe(obj):
"""
func join(items Array[Str]) Str ...
"""
if obj is None:
return []
# TODO: Need proper span IDs
if not isinstance(obj, str):
raise e_die('maybe() passed arg of invalid type %r',
obj.__class__.__name__)
s = obj
if len(s):
return [s]
else:
return []
def _StringToInteger(self, s, span_id=runtime.NO_SPID):
# type: (str, int) -> int
"""Use bash-like rules to coerce a string to an integer.
Runtime parsing enables silly stuff like $(( $(echo 1)$(echo 2) + 1 )) => 13
0xAB -- hex constant
042 -- octal constant
42 -- decimal constant
64#z -- arbitary base constant
bare word: variable
quoted word: string (not done?)
"""
if s.startswith('0x'):
try:
integer = int(s, 16)
except ValueError:
e_strict('Invalid hex constant %r', s, span_id=span_id)
return integer
if s.startswith('0'):
try:
integer = int(s, 8)
except ValueError:
e_strict('Invalid octal constant %r', s, span_id=span_id)
return integer
if '#' in s:
b, digits = mylib.split_once(s, '#')
try:
base = int(b)
except ValueError:
e_strict('Invalid base for numeric constant %r', b, span_id=span_id)
integer = 0
for ch in digits:
if IsLower(ch):
digit = ord(ch) - ord('a') + 10
elif IsUpper(ch):
digit = ord(ch) - ord('A') + 36
elif ch == '@': # horrible syntax
digit = 62
elif ch == '_':
digit = 63
elif ch.isdigit():
digit = int(ch)
else:
e_strict('Invalid digits for numeric constant %r', digits, span_id=span_id)
if digit >= base:
e_strict('Digits %r out of range for base %d', digits, base, span_id=span_id)
integer = integer * base + digit
return integer
try:
# Normal base 10 integer. This includes negative numbers like '-42'.
integer = int(s)
except ValueError:
# doesn't look like an integer
# note: 'test' and '[' never evaluate recursively
if self.exec_opts.eval_unsafe_arith() and self.parse_ctx:
# Special case so we don't get EOF error
if len(s.strip()) == 0:
return 0
# For compatibility: Try to parse it as an expression and evaluate it.
arena = self.parse_ctx.arena
a_parser = self.parse_ctx.MakeArithParser(s)
with alloc.ctx_Location(arena, source.Variable(span_id)):
try:
node2 = a_parser.Parse() # may raise error.Parse
except error.Parse as e:
ui.PrettyPrintError(e, arena)
e_die('Parse error in recursive arithmetic', span_id=e.span_id)
# Prevent infinite recursion of $(( 1x )) -- it's a word that evaluates
# to itself, and you don't want to reparse it as a word.
if node2.tag_() == arith_expr_e.Word:
e_die("Invalid integer constant %r", s, span_id=span_id)
else:
integer = self.EvalToInt(node2)
else:
if len(s.strip()) == 0 or match.IsValidVarName(s):
# x42 could evaluate to 0
e_strict("Invalid integer constant %r", s, span_id=span_id)
else:
# 42x is always fatal!
e_die("Invalid integer constant %r", s, span_id=span_id)
return integer
def _MaybeReplaceLeaf(self, node):
# type: (re_t) -> Tuple[Optional[re_t], bool]
"""
If a leaf node needs to be evaluated, do it and return the replacement.
Otherwise return None.
"""
new_leaf = None
recurse = True
if node.tag == re_e.Speck:
id_ = node.id
if id_ == Id.Expr_Dot:
new_leaf = re.Primitive(Id.Re_Dot)
elif id_ == Id.Arith_Caret: # ^
new_leaf = re.Primitive(Id.Re_Start)
elif id_ == Id.Expr_Dollar: # $
new_leaf = re.Primitive(Id.Re_End)
else:
raise NotImplementedError(id_)
elif node.tag == re_e.Token:
id_ = node.id
val = node.val
if id_ == Id.Expr_Name:
if val == 'dot':
new_leaf = re.Primitive(Id.Re_Dot)
else:
raise NotImplementedError(val)
elif id_ == Id.Expr_Symbol:
if val == '%start':
new_leaf = re.Primitive(Id.Re_Start)
elif val == '%end':
new_leaf = re.Primitive(Id.Re_End)
else:
raise NotImplementedError(val)
else: # Must be Id.Char_{OneChar,Hex,Unicode4,Unicode8}
kind = consts.GetKind(id_)
assert kind == Kind.Char, id_
s = word_compile.EvalCStringToken(node)
new_leaf = re.LiteralChars(s, node.span_id)
elif node.tag == re_e.SingleQuoted:
s = word_eval.EvalSingleQuoted(node)
new_leaf = re.LiteralChars(s, node.left.span_id)
elif node.tag == re_e.DoubleQuoted:
s = self.word_ev.EvalDoubleQuotedToString(node)
new_leaf = re.LiteralChars(s, node.left.span_id)
elif node.tag == re_e.BracedVarSub:
s = self.word_ev.EvalBracedVarSubToString(node)
new_leaf = re.LiteralChars(s, node.spids[0])
elif node.tag == re_e.SimpleVarSub:
s = self.word_ev.EvalSimpleVarSubToString(node.token)
new_leaf = re.LiteralChars(s, node.token.span_id)
elif node.tag == re_e.Splice:
obj = self.LookupVar(node.name.val)
if not isinstance(obj, objects.Regex):
e_die("Can't splice object of type %r into regex",
obj.__class__,
token=node.name)
# Note: we only splice the regex, and ignore flags.
# Should we warn about this?
new_leaf = obj.regex
# These are leaves we don't need to do anything with.
elif node.tag == re_e.PosixClass:
recurse = False
elif node.tag == re_e.PerlClass:
recurse = False
return new_leaf, recurse
def EvalExpr(self, node):
# type: (expr_t) -> Any
"""
This is a naive PyObject evaluator! It uses the type dispatch of the host
Python interpreter.
Returns:
A Python object of ANY type. Should be wrapped in value.Obj() for
storing in Mem.
"""
if 0:
print('EvalExpr()')
node.PrettyPrint()
print('')
if node.tag == expr_e.Const:
id_ = node.c.id
if id_ == Id.Expr_DecInt:
return int(node.c.val)
elif id_ == Id.Expr_BinInt:
return int(node.c.val, 2)
elif id_ == Id.Expr_OctInt:
return int(node.c.val, 8)
elif id_ == Id.Expr_HexInt:
return int(node.c.val, 16)
elif id_ == Id.Expr_Float:
return float(node.c.val)
elif id_ == Id.Expr_Null:
return None
elif id_ == Id.Expr_True:
return True
elif id_ == Id.Expr_False:
return False
elif id_ == Id.Expr_Name:
# for {name: 'bob'}
# Maybe also :Symbol?
return node.c.val
# NOTE: We could allow Ellipsis for a[:, ...] here, but we're not using
# it yet.
raise AssertionError(id_)
if node.tag == expr_e.Var:
return self.LookupVar(node.name.val)
if node.tag == expr_e.CommandSub:
id_ = node.left_token.id
# &(echo block literal)
if id_ == Id.Left_AmpParen:
return 'TODO: value.Block'
else:
stdout = self.shell_ex.RunCommandSub(node.child)
if id_ == Id.Left_AtParen: # @(seq 3)
strs = self.splitter.SplitForWordEval(stdout)
return strs
else:
return stdout
if node.tag == expr_e.ShArrayLiteral:
words = braces.BraceExpandWords(node.words)
strs = self.word_ev.EvalWordSequence(words)
#log('ARRAY LITERAL EVALUATED TO -> %s', strs)
return objects.StrArray(strs)
if node.tag == expr_e.DoubleQuoted:
# In an ideal world, I would *statically* disallow:
# - "$@" and "${array[@]}"
# - backticks like `echo hi`
# - $(( 1+2 )) and $[] -- although useful for refactoring
# - not sure: ${x%%} -- could disallow this
# - these enters the ArgDQ state: "${a:-foo bar}" ?
# But that would complicate the parser/evaluator. So just rely on
# strict_array to disallow the bad parts.
return self.word_ev.EvalDoubleQuotedToString(node)
if node.tag == expr_e.SingleQuoted:
return word_eval.EvalSingleQuoted(node)
if node.tag == expr_e.BracedVarSub:
return self.word_ev.EvalBracedVarSubToString(node)
if node.tag == expr_e.SimpleVarSub:
return self.word_ev.EvalSimpleVarSubToString(node.token)
if node.tag == expr_e.Unary:
child = self.EvalExpr(node.child)
if node.op.id == Id.Arith_Minus:
return -child
if node.op.id == Id.Arith_Tilde:
return ~child
if node.op.id == Id.Expr_Not:
return not child
raise NotImplementedError(node.op.id)
if node.tag == expr_e.Binary:
left = self.EvalExpr(node.left)
right = self.EvalExpr(node.right)
if node.op.id == Id.Arith_Plus:
return left + right
if node.op.id == Id.Arith_Minus:
return left - right
if node.op.id == Id.Arith_Star:
return left * right
if node.op.id == Id.Arith_Slash:
# NOTE: from __future__ import division changes 5/2!
# But just make it explicit.
return float(left) / right # floating point division
if node.op.id == Id.Expr_DSlash:
return left // right # integer divison
if node.op.id == Id.Arith_Percent:
return left % right
if node.op.id == Id.Arith_DStar: # Exponentiation
return left**right
if node.op.id == Id.Arith_DPlus:
# list or string concatenation
return left + right
# Bitwise
if node.op.id == Id.Arith_Amp:
return left & right
if node.op.id == Id.Arith_Pipe:
return left | right
if node.op.id == Id.Arith_Caret:
return left ^ right
if node.op.id == Id.Arith_DGreat:
return left >> right
if node.op.id == Id.Arith_DLess:
return left << right
# Logical
if node.op.id == Id.Expr_And:
return left and right
if node.op.id == Id.Expr_Or:
return left or right
raise NotImplementedError(node.op.id)
if node.tag == expr_e.Range: # 1:10 or 1:10:2
lower = self.EvalExpr(node.lower)
upper = self.EvalExpr(node.upper)
return xrange(lower, upper)
if node.tag == expr_e.Slice: # a[:0]
lower = self.EvalExpr(node.lower) if node.lower else None
upper = self.EvalExpr(node.upper) if node.upper else None
return slice(lower, upper)
if node.tag == expr_e.Compare:
left = self.EvalExpr(node.left)
result = True # Implicit and
for op, right_expr in zip(node.ops, node.comparators):
right = self.EvalExpr(right_expr)
if op.id == Id.Arith_Less:
result = left < right
elif op.id == Id.Arith_Great:
result = left > right
elif op.id == Id.Arith_GreatEqual:
result = left >= right
elif op.id == Id.Arith_LessEqual:
result = left <= right
elif op.id == Id.Arith_DEqual:
result = left == right
elif op.id == Id.Expr_In:
result = left in right
elif op.id == Id.Node_NotIn:
result = left not in right
elif op.id == Id.Expr_Is:
result = left is right
elif op.id == Id.Node_IsNot:
result = left is not right
elif op.id == Id.Expr_DTilde:
e_die('~~ not implemented')
elif op.id == Id.Expr_NotDTilde:
e_die('!~~ not implemented')
else:
try:
if op.id == Id.Arith_Tilde:
result = self._EvalMatch(left, right, True)
elif op.id == Id.Expr_NotTilde:
result = not self._EvalMatch(left, right, False)
else:
raise AssertionError(op.id)
except RuntimeError as e:
# Status 2 indicates a regex parse error. This is fatal in OSH but
# not in bash, which treats [[ like a command with an exit code.
e_die("Invalid regex %r",
right,
span_id=op.span_id,
status=2)
if not result:
return result
left = right
return result
if node.tag == expr_e.IfExp:
b = self.EvalExpr(node.test)
if b:
return self.EvalExpr(node.body)
else:
return self.EvalExpr(node.orelse)
if node.tag == expr_e.List:
return [self.EvalExpr(e) for e in node.elts]
if node.tag == expr_e.Tuple:
return tuple(self.EvalExpr(e) for e in node.elts)
if node.tag == expr_e.Dict:
# NOTE: some keys are expr.Const
keys = [self.EvalExpr(e) for e in node.keys]
values = []
for i, e in enumerate(node.values):
if e.tag == expr_e.Implicit:
v = self.LookupVar(keys[i]) # {name}
else:
v = self.EvalExpr(e)
values.append(v)
return dict(zip(keys, values))
if node.tag == expr_e.ListComp:
# TODO:
# - Consolidate with command_e.OilForIn in osh/cmd_eval.py?
# - Do I have to push a temp frame here?
# Hm... lexical or dynamic scope is an issue.
result = []
comp = node.generators[0]
obj = self.EvalExpr(comp.iter)
# TODO: Handle x,y etc.
iter_name = comp.lhs[0].name.val
if isinstance(obj, str):
e_die("Strings aren't iterable")
else:
it = obj.__iter__()
while True:
try:
loop_val = it.next() # e.g. x
except StopIteration:
break
self.mem.SetVar(lvalue.Named(iter_name), value.Obj(loop_val),
scope_e.LocalOnly)
if comp.cond:
b = self.EvalExpr(comp.cond)
else:
b = True
if b:
item = self.EvalExpr(node.elt) # e.g. x*2
result.append(item)
return result
if node.tag == expr_e.GeneratorExp:
comp = node.generators[0]
obj = self.EvalExpr(comp.iter)
# TODO: Support (x for x, y in ...)
iter_name = comp.lhs[0].name.val
it = obj.__iter__()
# TODO: There is probably a much better way to do this!
# The scope of the loop variable is wrong, etc.
def _gen():
while True:
try:
loop_val = it.next() # e.g. x
except StopIteration:
break
self.mem.SetVar(lvalue.Named(iter_name),
value.Obj(loop_val), scope_e.LocalOnly)
if comp.cond:
b = self.EvalExpr(comp.cond)
else:
b = True
if b:
item = self.EvalExpr(node.elt) # e.g. x*2
yield item
return _gen()
if node.tag == expr_e.Lambda:
raise NotImplementedError()
# This used to depend on cmd_ev, but we no longer have it.
#return objects.Lambda(node, None)
if node.tag == expr_e.FuncCall:
func = self.EvalExpr(node.func)
pos_args, named_args = self.EvalArgList(node.args)
ret = func(*pos_args, **named_args)
return ret
if node.tag == expr_e.Subscript:
obj = self.EvalExpr(node.obj)
index = self._EvalIndices(node.indices)
return obj[index]
# TODO: obj.method() should be separate
if node.tag == expr_e.Attribute: # obj.attr
o = self.EvalExpr(node.obj)
id_ = node.op.id
if id_ == Id.Expr_Dot:
name = node.attr.val
# TODO: Does this do the bound method thing we do NOT want?
return getattr(o, name)
if id_ == Id.Expr_RArrow: # d->key is like d['key']
name = node.attr.val
return o[name]
if id_ == Id.Expr_DColon: # StaticName::member
raise NotImplementedError(id_)
# TODO: We should prevent virtual lookup here? This is a pure static
# namespace lookup?
# But Python doesn't any hook for this.
# Maybe we can just check that it's a module? And modules don't lookup
# in a supertype or __class__, etc.
raise AssertionError(id_)
if node.tag == expr_e.RegexLiteral: # obj.attr
# TODO: Should this just be an object that ~ calls?
return objects.Regex(self.EvalRegex(node.regex))
if node.tag == expr_e.ArrayLiteral: # obj.attr
items = [self.EvalExpr(item) for item in node.items]
if items:
# Determine type at runtime? If we have something like @[(i) (j)]
# then we don't know its type until runtime.
first = items[0]
if isinstance(first, bool):
return objects.BoolArray(bool(x) for x in items)
elif isinstance(first, int):
return objects.IntArray(int(x) for x in items)
elif isinstance(first, float):
return objects.FloatArray(float(x) for x in items)
elif isinstance(first, str):
return objects.StrArray(str(x) for x in items)
else:
raise AssertionError(first)
else:
# TODO: Should this have an unknown type?
# What happens when you mutate or extend it? You have to make sure
# that the type tags match?
return objects.BoolArray(items)
raise NotImplementedError(node.__class__.__name__)
def Run(self, cmd_val):
# type: (cmd_value__Argv) -> int
"""
printf: printf [-v var] format [argument ...]
"""
attrs, arg_r = flag_spec.ParseCmdVal('printf', cmd_val)
arg = arg_types.printf(attrs.attrs)
fmt, fmt_spid = arg_r.ReadRequired2('requires a format string')
varargs, spids = arg_r.Rest2()
#log('fmt %s', fmt)
#log('vals %s', vals)
arena = self.parse_ctx.arena
if fmt in self.parse_cache:
parts = self.parse_cache[fmt]
else:
line_reader = reader.StringLineReader(fmt, arena)
# TODO: Make public
lexer = self.parse_ctx._MakeLexer(line_reader)
parser = _FormatStringParser(lexer)
with alloc.ctx_Location(arena, source.ArgvWord(fmt_spid)):
try:
parts = parser.Parse()
except error.Parse as e:
self.errfmt.PrettyPrintError(e)
return 2 # parse error
self.parse_cache[fmt] = parts
if 0:
print()
for part in parts:
part.PrettyPrint()
print()
out = [] # type: List[str]
arg_index = 0
num_args = len(varargs)
backslash_c = False
while True:
for part in parts:
UP_part = part
if part.tag_() == printf_part_e.Literal:
part = cast(printf_part__Literal, UP_part)
token = part.token
if token.id == Id.Format_EscapedPercent:
s = '%'
else:
s = word_compile.EvalCStringToken(token)
out.append(s)
elif part.tag_() == printf_part_e.Percent:
part = cast(printf_part__Percent, UP_part)
flags = [] # type: List[str]
if len(part.flags) > 0:
for flag_token in part.flags:
flags.append(flag_token.val)
width = -1 # nonexistent
if part.width:
if part.width.id in (Id.Format_Num, Id.Format_Zero):
width_str = part.width.val
width_spid = part.width.span_id
elif part.width.id == Id.Format_Star:
if arg_index < num_args:
width_str = varargs[arg_index]
width_spid = spids[arg_index]
arg_index += 1
else:
width_str = '' # invalid
width_spid = runtime.NO_SPID
else:
raise AssertionError()
try:
width = int(width_str)
except ValueError:
if width_spid == runtime.NO_SPID:
width_spid = part.width.span_id
self.errfmt.Print_("printf got invalid width %r" %
width_str,
span_id=width_spid)
return 1
precision = -1 # nonexistent
if part.precision:
if part.precision.id == Id.Format_Dot:
precision_str = '0'
precision_spid = part.precision.span_id
elif part.precision.id in (Id.Format_Num,
Id.Format_Zero):
precision_str = part.precision.val
precision_spid = part.precision.span_id
elif part.precision.id == Id.Format_Star:
if arg_index < num_args:
precision_str = varargs[arg_index]
precision_spid = spids[arg_index]
arg_index += 1
else:
precision_str = ''
precision_spid = runtime.NO_SPID
else:
raise AssertionError()
try:
precision = int(precision_str)
except ValueError:
if precision_spid == runtime.NO_SPID:
precision_spid = part.precision.span_id
self.errfmt.Print_(
'printf got invalid precision %r' %
precision_str,
span_id=precision_spid)
return 1
#log('index=%d n=%d', arg_index, num_args)
if arg_index < num_args:
s = varargs[arg_index]
word_spid = spids[arg_index]
arg_index += 1
else:
s = ''
word_spid = runtime.NO_SPID
typ = part.type.val
if typ == 's':
if precision >= 0:
s = s[:precision] # truncate
elif typ == 'q':
s = qsn.maybe_shell_encode(s)
elif typ == 'b':
# Process just like echo -e, except \c handling is simpler.
c_parts = [] # type: List[str]
lex = match.EchoLexer(s)
while True:
id_, tok_val = lex.Next()
if id_ == Id.Eol_Tok: # Note: This is really a NUL terminator
break
# TODO: add span_id from argv
tok = Token(id_, runtime.NO_SPID, tok_val)
p = word_compile.EvalCStringToken(tok)
# Unusual behavior: '\c' aborts processing!
if p is None:
backslash_c = True
break
c_parts.append(p)
s = ''.join(c_parts)
elif typ in 'diouxX' or part.type.id == Id.Format_Time:
try:
d = int(s)
except ValueError:
if len(s) >= 1 and s[0] in '\'"':
# TODO: utf-8 decode s[1:] to be more correct. Probably
# depends on issue #366, a utf-8 library.
# Note: len(s) == 1 means there is a NUL (0) after the quote..
d = ord(s[1]) if len(s) >= 2 else 0
elif part.type.id == Id.Format_Time and len(
s) == 0 and word_spid == runtime.NO_SPID:
# Note: No argument means -1 for %(...)T as in Bash Reference
# Manual 4.2 "If no argument is specified, conversion behaves
# as if -1 had been given."
d = -1
else:
if word_spid == runtime.NO_SPID:
# Blame the format string
blame_spid = part.type.span_id
else:
blame_spid = word_spid
self.errfmt.Print_(
'printf expected an integer, got %r' % s,
span_id=blame_spid)
return 1
if typ in 'di':
s = str(d)
elif typ in 'ouxX':
if d < 0:
e_die(
"Can't format negative number %d with %%%s",
d,
typ,
span_id=part.type.span_id)
if typ == 'u':
s = str(d)
elif typ == 'o':
s = mylib.octal(d)
elif typ == 'x':
s = mylib.hex_lower(d)
elif typ == 'X':
s = mylib.hex_upper(d)
elif part.type.id == Id.Format_Time:
# %(...)T
# Initialize timezone:
# `localtime' uses the current timezone information initialized
# by `tzset'. The function `tzset' refers to the environment
# variable `TZ'. When the exported variable `TZ' is present,
# its value should be reflected in the real environment
# variable `TZ' before call of `tzset'.
#
# Note: unlike LANG, TZ doesn't seem to change behavior if it's
# not exported.
#
# TODO: In Oil, provide an API that doesn't rely on libc's
# global state.
tzcell = self.mem.GetCell('TZ')
if tzcell and tzcell.exported and tzcell.val.tag_(
) == value_e.Str:
tzval = cast(value__Str, tzcell.val)
posix.putenv('TZ', tzval.s)
time_.tzset()
# Handle special values:
# User can specify two special values -1 and -2 as in Bash
# Reference Manual 4.2: "Two special argument values may be
# used: -1 represents the current time, and -2 represents the
# time the shell was invoked." from
# https://www.gnu.org/software/bash/manual/html_node/Bash-Builtins.html#index-printf
if d == -1: # the current time
ts = time_.time()
elif d == -2: # the shell start time
ts = self.shell_start_time
else:
ts = d
s = time_.strftime(typ[1:-2], time_.localtime(ts))
if precision >= 0:
s = s[:precision] # truncate
else:
raise AssertionError()
else:
raise AssertionError()
if width >= 0:
if len(flags):
if '-' in flags:
s = s.ljust(width, ' ')
elif '0' in flags:
s = s.rjust(width, '0')
else:
pass
else:
s = s.rjust(width, ' ')
out.append(s)
else:
raise AssertionError()
if backslash_c: # 'printf %b a\cb xx' - \c terminates processing!
break
if arg_index >= num_args:
break
# Otherwise there are more args. So cycle through the loop once more to
# implement the 'arg recycling' behavior.
result = ''.join(out)
if arg.v is not None:
# TODO: get the span_id for arg.v!
v_spid = runtime.NO_SPID
arena = self.parse_ctx.arena
a_parser = self.parse_ctx.MakeArithParser(arg.v)
with alloc.ctx_Location(arena, source.ArgvWord(v_spid)):
try:
anode = a_parser.Parse()
except error.Parse as e:
ui.PrettyPrintError(e, arena) # show parse error
e_usage('Invalid -v expression', span_id=v_spid)
lval = self.arith_ev.EvalArithLhs(anode, v_spid)
if not self.exec_opts.eval_unsafe_arith(
) and lval.tag_() != lvalue_e.Named:
e_usage(
'-v expected a variable name. shopt -s eval_unsafe_arith allows expressions',
span_id=v_spid)
state.SetRef(self.mem, lval, value.Str(result))
else:
mylib.Stdout().write(result)
return 0
def DoUnarySuffixOp(s, op, arg, extglob):
# type: (str, suffix_op__Unary, str, bool) -> str
"""Helper for ${x#prefix} and family."""
# Fast path for constant strings.
if not glob_.LooksLikeGlob(arg):
# It doesn't look like a glob, but we glob-escaped it (e.g. [ -> \[). So
# reverse it. NOTE: We also do this check in Globber.Expand(). It would
# be nice to somehow store the original string rather tahn
# escaping/unescaping.
arg = glob_.GlobUnescape(arg)
if op.op_id in (Id.VOp1_Pound, Id.VOp1_DPound): # const prefix
# explicit check for non-empty arg (len for mycpp)
if len(arg) and s.startswith(arg):
return s[len(arg):]
else:
return s
elif op.op_id in (Id.VOp1_Percent, Id.VOp1_DPercent): # const suffix
# need explicit check for non-empty arg (len for mycpp)
if len(arg) and s.endswith(arg):
return s[:-len(arg)]
else:
return s
# These operators take glob arguments, we don't implement that obscure case.
elif op.op_id == Id.VOp1_Comma: # Only lowercase the first letter
if arg != '':
# TODO: location info for op
e_die("%s can't have an argument", ui.PrettyId(op.op_id))
if len(s):
return s[0].lower() + s[1:]
else:
return s
elif op.op_id == Id.VOp1_DComma:
if arg != '':
e_die("%s can't have an argument", ui.PrettyId(op.op_id))
return s.lower()
elif op.op_id == Id.VOp1_Caret: # Only uppercase the first letter
if arg != '':
e_die("%s can't have an argument", ui.PrettyId(op.op_id))
if len(s):
return s[0].upper() + s[1:]
else:
return s
elif op.op_id == Id.VOp1_DCaret:
if arg != '':
e_die("%s can't have an argument", ui.PrettyId(op.op_id))
return s.upper()
else: # e.g. ^ ^^ , ,,
raise AssertionError(op.op_id)
# For patterns, do fnmatch() in a loop.
#
# TODO:
# - Another potential fast path:
# v=aabbccdd
# echo ${v#*b} # strip shortest prefix
#
# If the whole thing doesn't match '*b*', then no test can succeed. So we
# can fail early. Conversely echo ${v%%c*} and '*c*'.
#
# (Although honestly this whole construct is nuts and should be deprecated.)
n = len(s)
if op.op_id == Id.VOp1_Pound: # shortest prefix
# 'abcd': match '', 'a', 'ab', 'abc', ...
i = 0
while True:
assert i <= n
#log('Matching pattern %r with %r', arg, s[:i])
if libc.fnmatch(arg, s[:i], extglob):
return s[i:]
if i >= n:
break
i = _NextUtf8Char(s, i)
return s
elif op.op_id == Id.VOp1_DPound: # longest prefix
# 'abcd': match 'abc', 'ab', 'a'
i = n
while True:
assert i >= 0
#log('Matching pattern %r with %r', arg, s[:i])
if libc.fnmatch(arg, s[:i], extglob):
return s[i:]
if i == 0:
break
i = PreviousUtf8Char(s, i)
return s
elif op.op_id == Id.VOp1_Percent: # shortest suffix
# 'abcd': match 'abcd', 'abc', 'ab', 'a'
i = n
while True:
assert i >= 0
#log('Matching pattern %r with %r', arg, s[:i])
if libc.fnmatch(arg, s[i:], extglob):
return s[:i]
if i == 0:
break
i = PreviousUtf8Char(s, i)
return s
elif op.op_id == Id.VOp1_DPercent: # longest suffix
# 'abcd': match 'abc', 'bc', 'c', ...
i = 0
while True:
assert i <= n
#log('Matching pattern %r with %r', arg, s[:i])
if libc.fnmatch(arg, s[i:], extglob):
return s[:i]
if i >= n:
break
i = _NextUtf8Char(s, i)
return s
else:
raise NotImplementedError(ui.PrettyId(op.op_id))
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
"""
Run builtins, functions, external commands
Oil and other languages might have different, simpler rules. No special
builtins, etc.
Oil might have OIL_PATH = @( ... ) or something.
Interpreters might want to define all their own builtins.
Args:
procs: whether to look up procs.
"""
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(cmd_val.arg_spids) else runtime.NO_SPID
# This happens when you write "$@" but have no arguments.
if len(argv) == 0:
if self.exec_opts.strict_argv():
e_die("Command evaluated to an empty argv array", span_id=span_id)
else:
return 0 # status 0, or skip it?
arg0 = argv[0]
builtin_id = consts.LookupAssignBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
# command readonly is disallowed, for technical reasons. Could relax it
# later.
self.errfmt.Print_("Can't run assignment builtin recursively",
span_id=span_id)
return 1
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
status = self.RunBuiltin(builtin_id, cmd_val)
# TODO: Enable this and fix spec test failures.
# Also update _SPECIAL_BUILTINS in osh/builtin.py.
#if status != 0:
# e_die('special builtin failed', status=status)
return status
# TODO: if shopt -s namespaces, then look up in current namespace FIRST.
#
# Then fallback on self.procs, which should be renamed self.procs?
#
# honestly there is no real chance of colllision because
# foo-bar() {} can't be accessed anyway
# functions can have hyphens, but variables can't
# Builtins like 'true' can be redefined as functions.
if call_procs:
proc_node = self.procs.get(arg0)
if proc_node is not None:
if (self.exec_opts.strict_errexit() and
self.mutable_opts.ErrExitIsDisabled()):
self.errfmt.Print_('errexit was disabled for this construct',
span_id=self.mutable_opts.ErrExitSpanId())
self.errfmt.StderrLine('')
e_die("Can't run a proc while errexit is disabled. "
"Use 'run' or wrap it in a process with $0 myproc",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(proc_node, argv[1:])
return status
# TODO:
# look up arg0 in global namespace? And see if the type is value.Obj
# And it's a proc?
# isinstance(val.obj, objects.Proc)
UP_val = self.mem.GetVar(arg0)
if mylib.PYTHON: # Not reusing CPython objects
if UP_val.tag_() == value_e.Obj:
val = cast(value__Obj, UP_val)
if isinstance(val.obj, objects.Proc):
status = self.cmd_ev.RunOilProc(val.obj, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
environ = self.mem.GetExported() # Include temporary variables
if cmd_val.block:
e_die('Unexpected block passed to external command %r', arg0,
span_id=cmd_val.block.spids[0])
# Resolve argv[0] BEFORE forking.
argv0_path = self.search_path.CachedLookup(arg0)
if argv0_path is None:
self.errfmt.Print_('%r not found' % arg0, span_id=span_id)
return 127
# Normal case: ls /
if do_fork:
thunk = process.ExternalThunk(self.ext_prog, argv0_path, cmd_val, environ)
p = process.Process(thunk, self.job_state)
status = p.Run(self.waiter)
return status
# Already forked for pipeline: ls / | wc -l
# TODO: count subshell? ( ls / ) vs. ( ls /; ls / )
self.ext_prog.Exec(argv0_path, cmd_val, environ) # NEVER RETURNS
assert False, "This line should never be reached" # makes mypy happy
def RunCommandSub(self, cs_part):
# type: (command_sub) -> str
if not self.exec_opts.allow_command_sub():
# TODO: Add spid of $(
e_die("Command subs not allowed when errexit disabled (strict_errexit)")
node = cs_part.child
# Hack for weird $(<file) construct
if node.tag_() == command_e.Simple:
simple = cast(command__Simple, node)
# Detect '< file'
if (len(simple.words) == 0 and
len(simple.redirects) == 1 and
simple.redirects[0].op.id == Id.Redir_Less):
# change it to __cat < file
# note: cmd_eval.py _Dispatch works around lack of spid
tok = Token(Id.Lit_Chars, runtime.NO_SPID, '__cat')
cat_word = compound_word([tok])
# MUTATE the command.Simple node. This will only be done the first
# time in the parent process.
simple.words.append(cat_word)
p = self._MakeProcess(node,
inherit_errexit=self.exec_opts.inherit_errexit())
r, w = posix.pipe()
p.AddStateChange(process.StdoutToPipe(r, w))
_ = p.Start()
#log('Command sub started %d', pid)
chunks = [] # type: List[str]
posix.close(w) # not going to write
while True:
byte_str = posix.read(r, 4096)
if len(byte_str) == 0:
break
chunks.append(byte_str)
posix.close(r)
status = p.Wait(self.waiter)
# OSH has the concept of aborting in the middle of a WORD. We're not
# waiting until the command is over!
if self.exec_opts.command_sub_errexit():
if status != 0:
raise error.ErrExit(
'Command sub exited with status %d (%s)' %
(status, ui.CommandType(node)), span_id=cs_part.left_token.span_id,
status=status)
else:
# Set a flag so we check errexit at the same time as bash. Example:
#
# a=$(false)
# echo foo # no matter what comes here, the flag is reset
#
# Set ONLY until this command node has finished executing.
# HACK: move this
self.cmd_ev.check_command_sub_status = True
self.mem.SetLastStatus(status)
# Runtime errors test case: # $("echo foo > $@")
# Why rstrip()?
# https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
return ''.join(chunks).rstrip('\n')
def OldValue(lval, mem, exec_opts):
# type: (lvalue_t, Mem, optview.Exec) -> value_t
"""
Used by s+='x' and (( i += 1 ))
TODO: We need a stricter and less ambiguous version for Oil.
Problem:
- why does lvalue have Indexed and Keyed, while sh_lhs_expr only has
IndexedName?
- should I have lvalue.Named and lvalue.Indexed only?
- and Indexed uses the index_t type?
- well that might be Str or Int
"""
assert isinstance(lval, lvalue_t), lval
# TODO: refactor lvalue_t to make this simpler
UP_lval = lval
with tagswitch(lval) as case:
if case(lvalue_e.Named): # (( i++ ))
lval = cast(lvalue__Named, UP_lval)
var_name = lval.name
elif case(lvalue_e.Indexed): # (( a[i]++ ))
lval = cast(lvalue__Indexed, UP_lval)
var_name = lval.name
elif case(lvalue_e.Keyed): # (( A['K']++ )) ? I think this works
lval = cast(lvalue__Keyed, UP_lval)
var_name = lval.name
else:
raise AssertionError()
val = _LookupVar(var_name, mem, exec_opts)
UP_val = val
with tagswitch(lval) as case:
if case(lvalue_e.Named):
return val
elif case(lvalue_e.Indexed):
lval = cast(lvalue__Indexed, UP_lval)
array_val = None # type: value__MaybeStrArray
with tagswitch(val) as case2:
if case2(value_e.Undef):
array_val = value.MaybeStrArray([])
elif case2(value_e.MaybeStrArray):
tmp = cast(value__MaybeStrArray, UP_val)
# mycpp rewrite: add tmp. cast() creates a new var in inner scope
array_val = tmp
else:
e_die("Can't use [] on value of type %s", ui.ValType(val))
s = word_eval.GetArrayItem(array_val.strs, lval.index)
if s is None:
val = value.Str('') # NOTE: Other logic is value.Undef()? 0?
else:
assert isinstance(s, str), s
val = value.Str(s)
elif case(lvalue_e.Keyed):
lval = cast(lvalue__Keyed, UP_lval)
assoc_val = None # type: value__AssocArray
with tagswitch(val) as case2:
if case2(value_e.Undef):
# This never happens, because undef[x]+= is assumed to
raise AssertionError()
elif case2(value_e.AssocArray):
tmp2 = cast(value__AssocArray, UP_val)
# mycpp rewrite: add tmp. cast() creates a new var in inner scope
assoc_val = tmp2
else:
e_die("Can't use [] on value of type %s", ui.ValType(val))
s = assoc_val.d.get(lval.key)
if s is None:
val = value.Str('')
else:
val = value.Str(s)
else:
raise AssertionError()
return val
def EvalB(self, node):
# type: (bool_expr_t) -> bool
UP_node = node
with tagswitch(node) as case:
if case(bool_expr_e.WordTest):
node = cast(bool_expr__WordTest, UP_node)
s = self._EvalCompoundWord(node.w)
return bool(s)
elif case(bool_expr_e.LogicalNot):
node = cast(bool_expr__LogicalNot, UP_node)
b = self.EvalB(node.child)
return not b
elif case(bool_expr_e.LogicalAnd):
node = cast(bool_expr__LogicalAnd, UP_node)
# Short-circuit evaluation
if self.EvalB(node.left):
return self.EvalB(node.right)
else:
return False
elif case(bool_expr_e.LogicalOr):
node = cast(bool_expr__LogicalOr, UP_node)
if self.EvalB(node.left):
return True
else:
return self.EvalB(node.right)
elif case(bool_expr_e.Unary):
node = cast(bool_expr__Unary, UP_node)
op_id = node.op_id
s = self._EvalCompoundWord(node.child)
# Now dispatch on arg type
arg_type = consts.BoolArgType(op_id) # could be static in the LST?
if arg_type == bool_arg_type_e.Path:
return bool_stat.DoUnaryOp(op_id, s)
if arg_type == bool_arg_type_e.Str:
if op_id == Id.BoolUnary_z:
return not bool(s)
if op_id == Id.BoolUnary_n:
return bool(s)
raise AssertionError(op_id) # should never happen
if arg_type == bool_arg_type_e.Other:
if op_id == Id.BoolUnary_t:
return bool_stat.isatty(s, node.child)
# See whether 'set -o' options have been set
if op_id == Id.BoolUnary_o:
index = match.MatchOption(s)
if index == 0:
return False
else:
return self.exec_opts.opt_array[index]
if op_id == Id.BoolUnary_v:
val = self.mem.GetVar(s)
return val.tag_() != value_e.Undef
e_die("%s isn't implemented", ui.PrettyId(op_id)) # implicit location
raise AssertionError(arg_type) # should never happen
elif case(bool_expr_e.Binary):
node = cast(bool_expr__Binary, UP_node)
op_id = node.op_id
# Whether to glob escape
with switch(op_id) as case2:
if case2(Id.BoolBinary_GlobEqual, Id.BoolBinary_GlobDEqual,
Id.BoolBinary_GlobNEqual):
quote_kind = quote_e.FnMatch
elif case2(Id.BoolBinary_EqualTilde):
quote_kind = quote_e.ERE
else:
quote_kind = quote_e.Default
s1 = self._EvalCompoundWord(node.left)
s2 = self._EvalCompoundWord(node.right, quote_kind=quote_kind)
# Now dispatch on arg type
arg_type = consts.BoolArgType(op_id)
if arg_type == bool_arg_type_e.Path:
return bool_stat.DoBinaryOp(op_id, s1, s2)
if arg_type == bool_arg_type_e.Int:
# NOTE: We assume they are constants like [[ 3 -eq 3 ]].
# Bash also allows [[ 1+2 -eq 3 ]].
i1 = self._StringToIntegerOrError(s1, blame_word=node.left)
i2 = self._StringToIntegerOrError(s2, blame_word=node.right)
if op_id == Id.BoolBinary_eq:
return i1 == i2
if op_id == Id.BoolBinary_ne:
return i1 != i2
if op_id == Id.BoolBinary_gt:
return i1 > i2
if op_id == Id.BoolBinary_ge:
return i1 >= i2
if op_id == Id.BoolBinary_lt:
return i1 < i2
if op_id == Id.BoolBinary_le:
return i1 <= i2
raise AssertionError(op_id) # should never happen
if arg_type == bool_arg_type_e.Str:
if op_id in (Id.BoolBinary_GlobEqual, Id.BoolBinary_GlobDEqual):
#log('Matching %s against pattern %s', s1, s2)
return libc.fnmatch(s2, s1, self.exec_opts.extglob())
if op_id == Id.BoolBinary_GlobNEqual:
return not libc.fnmatch(s2, s1, self.exec_opts.extglob())
if op_id in (Id.BoolBinary_Equal, Id.BoolBinary_DEqual):
return s1 == s2
if op_id == Id.BoolBinary_NEqual:
return s1 != s2
if op_id == Id.BoolBinary_EqualTilde:
# TODO: This should go to --debug-file
#log('Matching %r against regex %r', s1, s2)
try:
matches = libc.regex_match(s2, s1)
except RuntimeError as e:
# Status 2 indicates a regex parse error. This is fatal in OSH but
# not in bash, which treats [[ like a command with an exit code.
msg = e.message # type: str
e_die("Invalid regex %r: %s", s2, msg, word=node.right,
status=2)
if matches is None:
return False
self._SetRegexMatches(matches)
return True
if op_id == Id.Op_Less:
return str_cmp(s1, s2) < 0
if op_id == Id.Op_Great:
return str_cmp(s1, s2) > 0
raise AssertionError(op_id) # should never happen
raise AssertionError(node.tag_())
def Run(self, cmd_val):
# type: (cmd_value__Argv) -> int
attrs, arg_r = flag_spec.ParseCmdVal('read', cmd_val)
arg = arg_types.read(attrs.attrs)
names = arg_r.Rest()
# Don't respect any of the other options here? This is buffered I/O.
if arg.line: # read --line
var_name, var_spid = arg_r.Peek2()
if var_name is None:
var_name = '_line'
else:
if var_name.startswith(':'): # optional : sigil
var_name = var_name[1:]
arg_r.Next()
next_arg, next_spid = arg_r.Peek2()
if next_arg is not None:
raise error.Usage('got extra argument', span_id=next_spid)
return self._Line(arg, var_name)
if arg.q:
e_usage('--qsn can only be used with --line')
if arg.all: # read --all
var_name, var_spid = arg_r.Peek2()
if var_name is None:
var_name = '_all'
else:
if var_name.startswith(':'): # optional : sigil
var_name = var_name[1:]
arg_r.Next()
next_arg, next_spid = arg_r.Peek2()
if next_arg is not None:
raise error.Usage('got extra argument', span_id=next_spid)
return self._All(var_name)
if arg.q:
e_usage('--qsn not implemented yet')
fd = self.stdin.fileno()
if arg.t >= 0.0:
if arg.t != 0.0:
e_die("read -t isn't implemented (except t=0)")
else:
return 0 if pyos.InputAvailable(fd) else 1
bits = 0
if self.stdin.isatty():
bits |= pyos.TERM_ICANON
if arg.s: # silent
bits |= pyos.TERM_ECHO
if arg.p is not None: # only if tty
mylib.Stderr().write(arg.p)
if bits == 0:
status = self._Read(arg, names)
else:
term = pyos.TermState(fd, ~bits)
try:
status = self._Read(arg, names)
finally:
term.Restore()
return status
def AsPosixEre(node, parts):
# type: (re_t, List[str]) -> None
"""Translate an Oil regex to a POSIX ERE.
Appends to a list of parts that you hvae to join.
"""
UP_node = node
tag = node.tag_()
if tag == re_e.Primitive:
node = cast(re__Primitive, UP_node)
if node.id == Id.Re_Dot:
parts.append('.')
elif node.id == Id.Re_Start:
parts.append('^')
elif node.id == Id.Re_End:
parts.append('$')
else:
raise AssertionError(node.id)
return
if tag == re_e.LiteralChars:
node = cast(re__LiteralChars, UP_node)
# The bash [[ x =~ "." ]] construct also has to do this
# TODO: What about \0 and unicode escapes?
# Those won't be as LiteralChars I don't think?
# Unless you put them there through \0
# Maybe DISALLOW those.
# "Unprintable chars should be written as \0 or \x00 or \u0000"
parts.append(glob_.ExtendedRegexEscape(node.s))
return
if tag == re_e.Seq:
node = cast(re__Seq, UP_node)
for c in node.children:
AsPosixEre(c, parts)
return
if tag == re_e.Alt:
node = cast(re__Alt, UP_node)
for i, c in enumerate(node.children):
if i != 0:
parts.append('|')
AsPosixEre(c, parts)
return
if tag == re_e.Repeat:
node = cast(re__Repeat, UP_node)
# 'foo' or "foo" or $x or ${x} evaluated to too many chars
if node.child.tag_() == re_e.LiteralChars:
child = cast(re__LiteralChars, node.child)
if len(child.s) > 1:
# Note: Other regex dialects have non-capturing groups since we don't
# need this.
e_die(
"POSIX EREs don't have groups without capture, so this node "
"needs () around it.",
span_id=child.spid)
AsPosixEre(node.child, parts)
op = node.op
op_tag = op.tag_()
UP_op = op
if op_tag == re_repeat_e.Op:
op = cast(re_repeat__Op, UP_op)
op_id = op.op.id
if op_id == Id.Arith_Plus:
parts.append('+')
elif op_id == Id.Arith_Star:
parts.append('*')
elif op_id == Id.Arith_QMark:
parts.append('?')
else:
raise AssertionError(op_id)
return
if op_tag == re_repeat_e.Num:
op = cast(re_repeat__Num, UP_op)
parts.append('{%s}' % op.times.val)
return
if op_tag == re_repeat_e.Range:
op = cast(re_repeat__Range, UP_op)
lower = op.lower.val if op.lower else ''
upper = op.upper.val if op.upper else ''
parts.append('{%s,%s}' % (lower, upper))
return
raise NotImplementedError(op_tag)
# Special case for familiarity: () is acceptable as a group in ERE
if tag in (re_e.Group, re_e.Capture):
node = cast(re__Group, UP_node)
parts.append('(')
AsPosixEre(node.child, parts)
parts.append(')')
return
if tag == re_e.PerlClass:
node = cast(perl_class, UP_node)
n = node.name
chars = PERL_CLASS[node.name] # looks like [:digit:]
if node.negated:
pat = '[^%s]' % chars
else:
pat = '[%s]' % chars
parts.append(pat)
return
if tag == re_e.PosixClass:
node = cast(posix_class, UP_node)
n = node.name # looks like 'digit'
if node.negated:
pat = '[^[:%s:]]' % n
else:
pat = '[[:%s:]]' % n
parts.append(pat)
return
if tag == re_e.ClassLiteral:
node = cast(re__ClassLiteral, UP_node)
parts.append('[')
if node.negated:
parts.append('^')
for term in node.terms:
_ClassLiteralToPosixEre(term, parts)
parts.append(']')
return
raise NotImplementedError(tag)
def Eval(self, node):
# type: (arith_expr_t) -> value_t
"""
Args:
node: arith_expr_t
Returns:
None for Undef (e.g. empty cell) TODO: Don't return 0!
int for Str
List[int] for MaybeStrArray
Dict[str, str] for AssocArray (TODO: Should we support this?)
NOTE: (( A['x'] = 'x' )) and (( x = A['x'] )) are syntactically valid in
bash, but don't do what you'd think. 'x' sometimes a variable name and
sometimes a key.
"""
# OSH semantics: Variable NAMES cannot be formed dynamically; but INTEGERS
# can. ${foo:-3}4 is OK. $? will be a compound word too, so we don't have
# to handle that as a special case.
UP_node = node
with tagswitch(node) as case:
if case(arith_expr_e.VarRef): # $(( x )) (can be array)
tok = cast(Token, UP_node)
return _LookupVar(tok.val, self.mem, self.exec_opts)
elif case(arith_expr_e.Word): # $(( $x )) $(( ${x}${y} )), etc.
w = cast(compound_word, UP_node)
return self.word_ev.EvalWordToString(w)
elif case(arith_expr_e.UnaryAssign): # a++
node = cast(arith_expr__UnaryAssign, UP_node)
op_id = node.op_id
old_int, lval = self._EvalLhsAndLookupArith(node.child)
if op_id == Id.Node_PostDPlus: # post-increment
new_int = old_int + 1
ret = old_int
elif op_id == Id.Node_PostDMinus: # post-decrement
new_int = old_int - 1
ret = old_int
elif op_id == Id.Arith_DPlus: # pre-increment
new_int = old_int + 1
ret = new_int
elif op_id == Id.Arith_DMinus: # pre-decrement
new_int = old_int - 1
ret = new_int
else:
raise AssertionError(op_id)
#log('old %d new %d ret %d', old_int, new_int, ret)
self._Store(lval, new_int)
return value.Int(ret)
elif case(arith_expr_e.BinaryAssign): # a=1, a+=5, a[1]+=5
node = cast(arith_expr__BinaryAssign, UP_node)
op_id = node.op_id
if op_id == Id.Arith_Equal:
# Don't really need a span ID here, because tdop.CheckLhsExpr should
# have done all the validation.
lval = self.EvalArithLhs(node.left, runtime.NO_SPID)
rhs_int = self.EvalToInt(node.right)
self._Store(lval, rhs_int)
return value.Int(rhs_int)
old_int, lval = self._EvalLhsAndLookupArith(node.left)
rhs = self.EvalToInt(node.right)
if op_id == Id.Arith_PlusEqual:
new_int = old_int + rhs
elif op_id == Id.Arith_MinusEqual:
new_int = old_int - rhs
elif op_id == Id.Arith_StarEqual:
new_int = old_int * rhs
elif op_id == Id.Arith_SlashEqual:
if rhs == 0:
e_die('Divide by zero') # TODO: location
new_int = old_int / rhs
elif op_id == Id.Arith_PercentEqual:
if rhs == 0:
e_die('Divide by zero') # TODO: location
new_int = old_int % rhs
elif op_id == Id.Arith_DGreatEqual:
new_int = old_int >> rhs
elif op_id == Id.Arith_DLessEqual:
new_int = old_int << rhs
elif op_id == Id.Arith_AmpEqual:
new_int = old_int & rhs
elif op_id == Id.Arith_PipeEqual:
new_int = old_int | rhs
elif op_id == Id.Arith_CaretEqual:
new_int = old_int ^ rhs
else:
raise AssertionError(op_id) # shouldn't get here
self._Store(lval, new_int)
return value.Int(new_int)
elif case(arith_expr_e.Unary):
node = cast(arith_expr__Unary, UP_node)
op_id = node.op_id
i = self.EvalToInt(node.child)
if op_id == Id.Node_UnaryPlus:
ret = i
elif op_id == Id.Node_UnaryMinus:
ret = -i
elif op_id == Id.Arith_Bang: # logical negation
ret = 1 if i == 0 else 0
elif op_id == Id.Arith_Tilde: # bitwise complement
ret = ~i
else:
raise AssertionError(op_id) # shouldn't get here
return value.Int(ret)
elif case(arith_expr_e.Binary):
node = cast(arith_expr__Binary, UP_node)
op_id = node.op_id
# Short-circuit evaluation for || and &&.
if op_id == Id.Arith_DPipe:
lhs = self.EvalToInt(node.left)
if lhs == 0:
rhs = self.EvalToInt(node.right)
ret = int(rhs != 0)
else:
ret = 1 # true
return value.Int(ret)
if op_id == Id.Arith_DAmp:
lhs = self.EvalToInt(node.left)
if lhs == 0:
ret = 0 # false
else:
rhs = self.EvalToInt(node.right)
ret = int(rhs != 0)
return value.Int(ret)
if op_id == Id.Arith_LBracket:
# NOTE: Similar to bracket_op_e.ArrayIndex in osh/word_eval.py
left = self.Eval(node.left)
UP_left = left
with tagswitch(left) as case:
if case(value_e.MaybeStrArray):
array_val = cast(value__MaybeStrArray, UP_left)
index = self.EvalToInt(node.right)
s = word_eval.GetArrayItem(array_val.strs, index)
elif case(value_e.AssocArray):
left = cast(value__AssocArray, UP_left)
key = self.EvalWordToString(node.right)
s = left.d.get(key)
else:
# TODO: Add error context
e_die('Expected array or assoc in index expression, got %s',
ui.ValType(left))
if s is None:
val = value.Undef() # type: value_t
else:
val = value.Str(s)
return val
if op_id == Id.Arith_Comma:
self.EvalToInt(node.left) # throw away result
ret = self.EvalToInt(node.right)
return value.Int(ret)
# Rest are integers
lhs = self.EvalToInt(node.left)
rhs = self.EvalToInt(node.right)
if op_id == Id.Arith_Plus:
ret = lhs + rhs
elif op_id == Id.Arith_Minus:
ret = lhs - rhs
elif op_id == Id.Arith_Star:
ret = lhs * rhs
elif op_id == Id.Arith_Slash:
if rhs == 0:
# TODO: Could also blame /
e_die('Divide by zero',
span_id=location.SpanForArithExpr(node.right))
ret = lhs / rhs
elif op_id == Id.Arith_Percent:
if rhs == 0:
# TODO: Could also blame /
e_die('Divide by zero',
span_id=location.SpanForArithExpr(node.right))
ret = lhs % rhs
elif op_id == Id.Arith_DStar:
# OVM is stripped of certain functions that are somehow necessary for
# exponentiation.
# Python/ovm_stub_pystrtod.c:21: PyOS_double_to_string: Assertion `0'
# failed.
if rhs < 0:
e_die("Exponent can't be less than zero") # TODO: error location
ret = 1
for i in xrange(rhs):
ret *= lhs
elif op_id == Id.Arith_DEqual:
ret = int(lhs == rhs)
elif op_id == Id.Arith_NEqual:
ret = int(lhs != rhs)
elif op_id == Id.Arith_Great:
ret = int(lhs > rhs)
elif op_id == Id.Arith_GreatEqual:
ret = int(lhs >= rhs)
elif op_id == Id.Arith_Less:
ret = int(lhs < rhs)
elif op_id == Id.Arith_LessEqual:
ret = int(lhs <= rhs)
elif op_id == Id.Arith_Pipe:
ret = lhs | rhs
elif op_id == Id.Arith_Amp:
ret = lhs & rhs
elif op_id == Id.Arith_Caret:
ret = lhs ^ rhs
# Note: how to define shift of negative numbers?
elif op_id == Id.Arith_DLess:
ret = lhs << rhs
elif op_id == Id.Arith_DGreat:
ret = lhs >> rhs
else:
raise AssertionError(op_id)
return value.Int(ret)
elif case(arith_expr_e.TernaryOp):
node = cast(arith_expr__TernaryOp, UP_node)
cond = self.EvalToInt(node.cond)
if cond: # nonzero
return self.Eval(node.true_expr)
else:
return self.Eval(node.false_expr)
else:
raise AssertionError(node.tag_())
