def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
if action == 'py': # Prints the module
# Called by asdl/run.sh py-cpp
schema_path = argv[2]
app_types = {'id': asdl.UserType('id_kind_asdl', 'Id_t')}
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
root = sys.modules[__name__]
# NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
# demo.
py_meta.MakeTypes(schema_ast, root, type_lookup)
log('AST for this ASDL schema:')
schema_ast.Print(sys.stdout, 0)
print()
log('Dynamically created a Python module with these types:')
for name in dir(root):
print('\t' + name)
if 1:
# NOTE: It can be pickled, but not marshaled
import marshal
import cPickle
print(dir(marshal))
out_path = schema_path + '.pickle'
with open(out_path, 'w') as f:
#marshal.dump(type_lookup, f)
# Version 2 is the highest protocol for Python 2.7.
cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2)
print('runtime_type_lookup:')
for name, desc in type_lookup.runtime_type_lookup.items():
print(name)
print(desc)
print()
print('Wrote %s' % out_path)
elif action == 'arith-format': # pretty printing
expr = argv[2]
obj = typed_arith_parse.ParseShell(expr)
tree = obj.PrettyTree()
#treee= ['hi', 'there', ['a', 'b'], 'c']
f = fmt.DetectConsoleOutput(sys.stdout)
fmt.PrintTree(tree, f)
print()
# Might need to print the output?
# out.WriteToFile?
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
# type: (List[str]) -> int
try:
return AppBundleMain(argv)
except error.Usage as e:
#builtin.Help(['oil-usage'], util.GetResourceLoader())
log('oil: %s', e.msg)
return 2
except RuntimeError as e:
if 0:
import traceback
traceback.print_exc()
# NOTE: The Python interpreter can cause this, e.g. on stack overflow.
log('FATAL: %r', e)
return 1
except KeyboardInterrupt:
print()
return 130 # 128 + 2
except (IOError, OSError) as e:
if 0:
import traceback
traceback.print_exc()
# test this with prlimit --nproc=1 --pid=$$
stderr_line('osh I/O error: %s', posix.strerror(e.errno))
return 2 # dash gives status 2
finally:
_tlog('Exiting main()')
if _trace_path:
_tracer.Stop(_trace_path)
def testBraceRangeLexer(self):
lex = match.BraceRangeLexer('1..3')
while True:
id_, val = lex.Next()
log('%s %r', Id_str(id_), val)
if id_ == Id.Eol_Tok:
break
def testIntOverflow(self):
log('OVERFLOW')
CASES = [
0,
2**31,
2**32,
2**64 - 1,
2**64,
2**128,
]
for i in CASES:
print('--')
# This raises Overflow? I guess the problem is that yajl takes an
# integer.
#print(yajl.dumps(i))
s = str(i)
print(s)
print(yajl.loads('{"k": %d}' % i))
# Why doesn't it parse raw integers?
#print(yajl.loads(s))
log('')
def Next(self):
while True:
self.tok_id, self.tok_val, self.pos = self.tokens.next()
if self.tok_id not in ('Comment', 'Whitespace'):
break
if 0:
log('%s %r', self.tok_id, self.tok_val)
def EvalRegex(self, node):
# type: (re_t) -> re_t
"""
Resolve the references in an eggex, e.g. Hex and $const in
/ Hex '.' $const "--$const" /
"""
# Regex Evaluation Shares the Same Structure, but uses slightly different
# nodes.
# * Speck/Token (syntactic concepts) -> Primitive (logical)
# * Splice -> Resolved
# * All Strings -> Literal
new_leaf, recurse = self._MaybeReplaceLeaf(node)
if new_leaf:
return new_leaf
elif recurse:
self._MutateSubtree(node)
# View it after evaluation
if 0:
log('After evaluation:')
node.PrettyPrint()
print()
return node
def Run(self, cmd_val):
arg_r = args.Reader(cmd_val.argv, spids=cmd_val.arg_spids)
arg_r.Next() # skip 'use'
# TODO:
# - Does shopt -s namespaces have to be on?
# - I don't think so? It only affects 'procs', not funcs.
arg = arg_r.Peek()
# 'use bin' and 'use env' are for static analysis. No-ops at runtime.
if arg in ('bin', 'env'):
return 0
if arg == 'lib': # OPTIONAL lib
arg_r.Next()
# Cosmetic: separator for 'use bin __ grep sed'. Allowed for 'lib' to be
# consistent.
arg = arg_r.Peek()
if arg == '__': # OPTIONAL __
arg_r.Next()
# Now import everything.
rest = arg_r.Rest()
for path in rest:
log('path %s', path)
return 0
def Pop(self):
# type: () -> None
frame = self.stack.pop()
#log('< Pop %s', frame)
for rf in reversed(frame.saved):
if rf.saved_fd == NO_FD:
#log('Close %d', orig)
try:
posix.close(rf.orig_fd)
except OSError as e:
log('Error closing descriptor %d: %s', rf.orig_fd,
pyutil.strerror(e))
raise
else:
try:
posix.dup2(rf.saved_fd, rf.orig_fd)
except OSError as e:
log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
pyutil.strerror(e))
#log('fd state:')
#posix.system('ls -l /proc/%s/fd' % posix.getpid())
raise
posix.close(rf.saved_fd)
#log('dup2 %s %s', saved, orig)
# Wait for here doc processes to finish.
for proc, waiter in frame.need_wait:
unused_status = proc.Wait(waiter)
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(
cmd_val.arg_spids) else runtime.NO_SPID
arg0 = argv[0]
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# Copied from core/executor.py
if call_procs:
proc_node = self.procs.get(arg0)
if proc_node is not None:
if (self.exec_opts.strict_errexit()
and self.mutable_opts.ErrExitIsDisabled()):
# TODO: make errfmt a member
#self.errfmt.Print_('errexit was disabled for this construct',
# span_id=self.mutable_opts.errexit.spid_stack[0])
#stderr_line('')
e_die(
"Can't run a proc while errexit is disabled. "
"Use 'catch' or wrap it in a process with $0 myproc",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(proc_node, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# See how many tests will pass
#if mylib.PYTHON:
if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling
import subprocess
try:
status = subprocess.call(cmd_val.argv)
except OSError as e:
log('Error running %s: %s', cmd_val.argv, e)
return 1
return status
log('Unhandled SimpleCommand')
f = mylib.Stdout()
#ast_f = fmt.DetectConsoleOutput(f)
# Stupid Eclipse debugger doesn't display ANSI
ast_f = fmt.TextOutput(f)
tree = cmd_val.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
return 0
def testRead(self):
if posix_.environ.get('EINTR_TEST'):
# Now we can do kill -TERM PID can get EINTR.
# Or Ctrl-C for KeyboardInterrupt
signal.signal(signal.SIGTERM, _Handler)
log('Hanging on read in pid %d', posix_.getpid())
posix_.read(0, 1)
def testDict(self):
log('DICT')
d = {"bool": False, "int": 42, "float": 3.14, "string": "s"}
print(yajl.dumps(d))
s = '{"bool": false, "int": 42, "float": 3.14, "string": "s"}'
print(yajl.loads(s))
log('')
def testFloat(self):
log('FLOAT')
print(yajl.dumps(123.4))
# Bug fix over latest version of py-yajl: a lone float decodes
decoded = yajl.loads('123.4')
self.assertEqual(123.4, decoded)
log('')
def GetLineSpan(self, span_id):
# type: (int) -> line_span
assert span_id != runtime.NO_SPID, span_id
try:
return self.spans[span_id]
except IndexError:
log('Span ID out of range: %d is greater than %d', span_id,
len(self.spans))
raise
def RunSimpleCommand(self, cmd_val, do_fork, call_procs=True):
# type: (cmd_value__Argv, bool, bool) -> int
argv = cmd_val.argv
span_id = cmd_val.arg_spids[0] if len(
cmd_val.arg_spids) else runtime.NO_SPID
arg0 = argv[0]
builtin_id = consts.LookupSpecialBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
func_node = self.procs.get(arg0)
if func_node is not None:
if (self.exec_opts.strict_errexit()
and self.mutable_opts.errexit.SpidIfDisabled() !=
runtime.NO_SPID):
# NOTE: This would be checked below, but this gives a better error
# message.
e_die(
"can't disable errexit running a function. "
"Maybe wrap the function in a process with the at-splice "
"pattern.",
span_id=span_id)
# NOTE: Functions could call 'exit 42' directly, etc.
status = self.cmd_ev.RunProc(func_node, argv[1:])
return status
builtin_id = consts.LookupNormalBuiltin(arg0)
if builtin_id != consts.NO_INDEX:
return self.RunBuiltin(builtin_id, cmd_val)
# See how many tests will pass
#if mylib.PYTHON:
if 0: # osh_eval.cc will pass 1078 rather than 872 by enabling
import subprocess
try:
status = subprocess.call(cmd_val.argv)
except OSError as e:
log('Error running %s: %s', cmd_val.argv, e)
return 1
return status
log('Unhandled SimpleCommand')
f = mylib.Stdout()
#ast_f = fmt.DetectConsoleOutput(f)
# Stupid Eclipse debugger doesn't display ANSI
ast_f = fmt.TextOutput(f)
tree = cmd_val.PrettyTree()
ast_f.FileHeader()
fmt.PrintTree(tree, ast_f)
ast_f.FileFooter()
ast_f.write('\n')
return 0
def testWaitpid(self):
if posix_.environ.get('EINTR_TEST'):
# Now we can do kill -TERM PID can get EINTR.
signal.signal(signal.SIGTERM, _Handler)
p = subprocess.Popen(['sleep', '5'])
log('started sleep pid %d', p.pid)
log('Hanging on waitpid in pid %d', posix_.getpid())
posix_.waitpid(-1, 0)
def main(argv):
try:
sys.exit(AppBundleMain(argv))
except error.Usage as e:
#print(_OPY_USAGE, file=sys.stderr)
log('opy: %s', e.msg)
sys.exit(2)
except RuntimeError as e:
log('FATAL: %s', e)
sys.exit(1)
def PrintFuncs(funcs):
banner('FUNCTIONS')
funcs.sort() # sort by module name
import collections
type_hist = collections.Counter()
# 316 globals / constants (513 before deduping)
for (mod_name, name, obj) in funcs:
log('%-20s %-15s %r', mod_name, name, obj)
def testStringEncoding(self):
log('STRING ENCODE')
# It should just raise with Unicode instance
#print(yajl.dumps(u'abc\u0100def'))
# It inserts \xff literally, OK I guess that's fine. It's not valid utf-8
print(yajl.dumps('\x00\xff'))
# mu character
print(yajl.dumps('\xCE\xBC'))
def SplitForWordEval(self, s, ifs=None):
# type: (str, str) -> List[str]
"""
Split used by word evaluation. Also used by the explicit @split() functino.
"""
sp = self._GetSplitter(ifs=ifs)
spans = sp.Split(s, True)
if 0:
for span in spans:
log('SPAN %s', span)
return _SpansToParts(s, spans)
def _ParseOsh(self, code_str):
"""Parse a line of OSH, which can include Oil assignments."""
line_reader = reader.StringLineReader(code_str, self.arena)
# the OSH parser hooks into the Oil parser
c_parser = self.parse_ctx.MakeOshParser(line_reader)
node = c_parser.ParseLogicalLine()
print('')
log('\t%s', code_str)
node.PrettyPrint()
print('')
return node
def _Visit(self, node):
"""
"""
#log('VISIT %s', node.__class__.__name__)
# NOTE: The tags are not unique!!! We would need this:
# if isinstance(node, ast.command) and node.tag == command_e.Simple:
# But it's easier to check the __class__ attribute.
cls = node.__class__
if cls is command.Simple:
#log('SimpleCommand %s', node.words)
#log('--')
#node.PrettyPrint()
# Things to consider:
# - source and .
# - DONE builtins: get a list from builtin.py
# - DONE functions: have to enter function definitions into a dictionary
# - Commands that call others: sudo, su, find, xargs, etc.
# - builtins that call others: exec, command
# - except not command -v!
if not node.words:
return
w = node.words[0]
ok, argv0, _ = word_.StaticEval(w)
if not ok:
log("Couldn't statically evaluate %r", w)
return
if (consts.LookupSpecialBuiltin(argv0) == consts.NO_INDEX
and consts.LookupAssignBuiltin(argv0) == consts.NO_INDEX
and consts.LookupNormalBuiltin(argv0) == consts.NO_INDEX):
self.progs_used[argv0] = True
# NOTE: If argv1 is $0, then we do NOT print a warning!
if argv0 == 'sudo':
if len(node.words) < 2:
return
w1 = node.words[1]
ok, argv1, _ = word_.StaticEval(w1)
if not ok:
log("Couldn't statically evaluate %r", w)
return
# Should we mark them behind 'sudo'? e.g. "sudo apt install"?
self.progs_used[argv1] = True
elif cls is command.ShFunction:
self.funcs_defined[node.name] = True
def MaybeRemove(self, pid):
# type: (int) -> None
"""Process and Pipeline can call this."""
# Problem: This only happens after an explicit wait()?
# I think the main_loop in bash waits without blocking?
log('JobState MaybeRemove %d', pid)
# TODO: Enabling this causes a failure in spec/background.
return
try:
del self.jobs[pid]
except KeyError:
# This should never happen?
log("AssertionError: PID %d should have never been in the job list", pid)
def testInt(self):
log('INT')
encoded = yajl.dumps(123)
print('encoded = %r' % encoded)
self.assertEqual('123', encoded)
# Bug fix over latest version of py-yajl: a lone int decodes
decoded = yajl.loads('123\n')
print('decoded = %r' % decoded)
self.assertEqual(123, decoded)
decoded = yajl.loads('{"a":123}\n')
print('decoded = %r' % decoded)
log('')
def testMatchOption(self):
log('MatchOption')
CASES = [
('', False),
('pipefail', True),
('foo', False),
('pipefai', False),
('pipefail_', False),
('strict_errexit', True),
]
for s, expected_bool in CASES:
result = fastlex.MatchOption(s)
self.assertEqual(expected_bool, bool(result))
log('case %r, result = %s', s, result)
def GenBuiltinLookup(b, func_name, kind, f):
#log('%r %r', func_name, kind)
pairs = [(b.name, b.index) for b in _BUILTINS if b.kind == kind]
#log('%s', pairs)
groups = collections.defaultdict(list)
for name, index in pairs:
first_char = name[0]
groups[first_char].append((name, index))
if 0:
for first_char, pairs in groups.iteritems():
log('%s %d', first_char, len(pairs))
log('%s', pairs)
# Note: we could optimize the length check, e.g. have a second level
# switch. But we would need to measure the difference. Caching the id on
# AST nodes is probably a bigger win, e.g. for loops.
#
# Size optimization: don't repeat constants literally?
f.write("""\
builtin_t %s(Str* s) {
int len = s->len_;
if (len == 0) return 0; // consts.NO_INDEX
const char* data = s->data_;
switch (data[0]) {
""" % func_name)
for first_char in sorted(groups):
pairs = groups[first_char]
f.write(" case '%s':\n" % first_char)
for name, index in pairs:
# NOTE: we have to check the length because they're not NUL-terminated
f.write('''\
if (len == %d && memcmp("%s", data, %d) == 0) return %d;
''' % (len(name), name, len(name), index))
f.write(' break;\n')
f.write("""\
}
return 0; // consts.NO_INDEX
}
""")
def testPipeline(self):
node = _CommandNode('uniq -c', _ARENA)
cmd_ev = test_lib.InitCommandEvaluator(arena=_ARENA, ext_prog=_EXT_PROG)
print('BEFORE', os.listdir('/dev/fd'))
p = process.Pipeline()
p.Add(_ExtProc(['ls']))
p.Add(_ExtProc(['cut', '-d', '.', '-f', '2']))
p.Add(_ExtProc(['sort']))
p.AddLast((cmd_ev, node))
pipe_status = p.Run(_WAITER, _FD_STATE)
log('pipe_status: %s', pipe_status)
print('AFTER', os.listdir('/dev/fd'))
def testPipeline(self):
node = _CommandNode('uniq -c', self.arena)
cmd_ev = test_lib.InitCommandEvaluator(arena=self.arena,
ext_prog=self.ext_prog)
print('BEFORE', os.listdir('/dev/fd'))
p = process.Pipeline()
p.Add(self._ExtProc(['ls']))
p.Add(self._ExtProc(['cut', '-d', '.', '-f', '2']))
p.Add(self._ExtProc(['sort']))
p.AddLast((cmd_ev, node))
pipe_status = p.Run(self.waiter, self.fd_state)
log('pipe_status: %s', pipe_status)
print('AFTER', os.listdir('/dev/fd'))
def testProcess(self):
# 3 fds. Does Python open it? Shell seems to have it too. Maybe it
# inherits from the shell.
print('FDS BEFORE', os.listdir('/dev/fd'))
Banner('date')
p = _ExtProc(['date'])
status = p.Run(_WAITER)
log('date returned %d', status)
self.assertEqual(0, status)
Banner('does-not-exist')
p = _ExtProc(['does-not-exist'])
print(p.Run(_WAITER))
# 12 file descriptors open!
print('FDS AFTER', os.listdir('/dev/fd'))
def main(argv):
# type: (List[str]) -> int
loader = pyutil.GetResourceLoader()
login_shell = False
environ = {} # type: Dict[str, str]
environ['PWD'] = posix.getcwd()
arg_r = args.Reader(argv, spids=[runtime.NO_SPID] * len(argv))
try:
status = pure.Main('osh', arg_r, environ, login_shell, loader, None)
return status
except error.Usage as e:
#builtin.Help(['oil-usage'], util.GetResourceLoader())
log('oil: %s', e.msg)
return 2
except RuntimeError as e:
if 0:
import traceback
traceback.print_exc()
# NOTE: The Python interpreter can cause this, e.g. on stack overflow.
# f() { f; }; f will cause this
msg = e.message # type: str
stderr_line('osh fatal error: %s', msg)
return 1
# Note: This doesn't happen in C++.
except KeyboardInterrupt:
print('')
return 130 # 128 + 2
except OSError as e:
if 0:
import traceback
traceback.print_exc()
# test this with prlimit --nproc=1 --pid=$$
stderr_line('osh I/O error: %s', pyutil.strerror_OS(e))
return 2 # dash gives status 2
except IOError as e: # duplicate of above because CPython is inconsistent
stderr_line('osh I/O error: %s', pyutil.strerror_IO(e))
return 2
def MaybeDump(self, status):
# type: (int) -> None
"""Write the dump as JSON.
User can configure it two ways:
- dump unconditionally -- a daily cron job. This would be fine.
- dump on non-zero exit code
OIL_FAIL
Maybe counters are different than failure
OIL_CRASH_DUMP='function alias trap completion stack' ?
OIL_COUNTER_DUMP='function alias trap completion'
and then
I think both of these should dump the (path, mtime, checksum) of the source
they ran? And then you can match those up with source control or whatever?
"""
if not self.collected:
return
if mylib.PYTHON: # can't translate due to open()
my_pid = posix.getpid() # Get fresh PID here
# Other things we need: the reason for the crash! _ErrorWithLocation is
# required I think.
d = {
'var_stack': self.var_stack,
'argv_stack': self.argv_stack,
'debug_stack': self.debug_stack,
'error': self.error,
'status': status,
'pid': my_pid,
}
# TODO: Add PID here
path = os_path.join(self.crash_dump_dir,
'%d-osh-crash-dump.json' % my_pid)
with open(path, 'w') as f:
import json
json.dump(d, f, indent=2)
#print(repr(d), file=f)
log('[%d] Wrote crash dump to %s', my_pid, path)