def _input_reader_handler(self):
"""
:param cparser.State state:
:return: yields chars
:rtype: typing.Generator[str]
"""
state = self.state
old_err_num = len(state._errors)
old_content_list_num = len(state.contentlist)
while True:
try:
line = input(">>> ")
except EOFError:
break
for c in line + "\n":
yield c
for m in state._errors[old_err_num:]:
print("Error:", m)
old_err_num = len(state._errors)
for m in state.contentlist[old_content_list_num:]:
if self.debug:
print("Parsed:", m)
if isinstance(m, (cparser.CStatement, cparser.CControlStructureBase)):
try:
res = self.interp.runSingleStatement(m, dump=self.debug)
print(res)
except Exception as exc:
print("Interpreter exception:", type(exc).__name__, ":", exc)
if self.debug:
better_exchook.better_exchook(*sys.exc_info())
old_content_list_num = len(state.contentlist)
def excepthook(exc_type, exc_obj, exc_tb):
"""
:param exc_type:
:param exc_obj:
:param exc_tb:
"""
# noinspection PyBroadException
try:
# noinspection PyUnresolvedReferences,PyProtectedMember
is_main_thread = isinstance(threading.currentThread(), threading._MainThread)
except Exception: # Can happen at a very late state while quitting.
if exc_type is KeyboardInterrupt:
return
else:
if is_main_thread:
if exc_type is KeyboardInterrupt and getattr(sys, "exited", False):
# Got SIGINT twice. Can happen.
return
# An unhandled exception in the main thread. This means that we are going to quit now.
sys.exited = True
print("Unhandled exception %s in thread %s, proc %i." % (exc_type, threading.currentThread(), os.getpid()))
if exc_type is KeyboardInterrupt:
return
# noinspection PyUnresolvedReferences,PyProtectedMember
if isinstance(threading.currentThread(), threading._MainThread):
main_thread_id = thread.get_ident()
if not isinstance(exc_type, Exception):
# We are the main thread and we got an exit-exception. This is likely fatal.
# This usually means an exit. (We ignore non-daemon threads and procs here.)
# Print the stack of all other threads.
dump_all_thread_tracebacks(exclude_thread_ids={main_thread_id})
better_exchook.better_exchook(exc_type, exc_obj, exc_tb, file=sys.stdout)
def log_exception(where, exctype, value, traceback):
with lock:
print("Exception at %s" % where)
better_exchook.better_exchook(exctype,
value,
traceback,
autodebugshell=False)
def __del__(self): try: if self._has_opened_file(): self._close_file() except: print "SimpleStruct del error:", self, getattr(self, "_intern", None) better_exchook.better_exchook(*sys.exc_info())
def excepthook(exc_type, exc_obj, exc_tb):
try:
is_main_thread = isinstance(threading.currentThread(), threading._MainThread)
except Exception: # Can happen at a very late state while quitting.
if exc_type is KeyboardInterrupt:
return
else:
if is_main_thread:
if exc_type is KeyboardInterrupt and getattr(sys, "exited", False):
# Got SIGINT twice. Can happen.
return
# An unhandled exception in the main thread. This means that we are going to quit now.
sys.exited = True
print "Unhandled exception %s in thread %s, proc %i." % (exc_type, threading.currentThread(), os.getpid())
if exc_type is KeyboardInterrupt:
return
if isinstance(threading.currentThread(), threading._MainThread):
main_thread_id = thread.get_ident()
if not isinstance(exc_type, Exception):
# We are the main thread and we got an exit-exception. This is likely fatal.
# This usually means an exit. (We ignore non-daemon threads and procs here.)
# Print the stack of all other threads.
dumpAllThreadTracebacks({main_thread_id})
better_exchook.better_exchook(exc_type, exc_obj, exc_tb)
def excepthook(exc_type, exc_obj, exc_tb):
try:
is_main_thread = isinstance(threading.currentThread(),
threading._MainThread)
except Exception: # Can happen at a very late state while quitting.
if exc_type is KeyboardInterrupt:
return
else:
if is_main_thread:
if exc_type is KeyboardInterrupt and getattr(
sys, "exited", False):
# Got SIGINT twice. Can happen.
return
# An unhandled exception in the main thread. This means that we are going to quit now.
sys.exited = True
print("Unhandled exception %s in thread %s, proc %i." %
(exc_type, threading.currentThread(), os.getpid()))
if exc_type is KeyboardInterrupt:
return
if isinstance(threading.currentThread(), threading._MainThread):
main_thread_id = thread.get_ident()
if not isinstance(exc_type, Exception):
# We are the main thread and we got an exit-exception. This is likely fatal.
# This usually means an exit. (We ignore non-daemon threads and procs here.)
# Print the stack of all other threads.
dumpAllThreadTracebacks({main_thread_id})
better_exchook.better_exchook(exc_type, exc_obj, exc_tb)
def log_except(*args, **kw):
logger = kw.get('logger', None)
better = kw.pop('_better', 0)
if not logger:
logger = logging.root
if not len(args):
msg = None
elif len(args) == 1:
msg = args[0]
args = []
else:
msg = args[0]
args = args[1:]
lines = ['Traceback (most recent call last):\n']
if better:
import better_exchook
better_exchook.better_exchook(
*sys.exc_info(), output=lambda s: lines.append('%s\n' % s))
else:
ei = sys.exc_info()
st = traceback.extract_stack(f=ei[2].tb_frame.f_back)
et = traceback.extract_tb(ei[2])
lines.extend(traceback.format_list(st))
lines.append(' ****** Traceback ****** \n')
lines.extend(traceback.format_list(et))
lines.extend(traceback.format_exception_only(ei[0], ei[1]))
exc = ''.join(lines)
if msg:
args = list(args)
args.append(exc)
logger.error(msg + ':\n%s', *args)
else:
logger.error(exc)
def log_except(*args, **kw):
logger = kw.get('logger', None)
better = kw.pop('_better', 0)
if not logger:
logger = logging.root
if not len(args):
msg = None
elif len(args) == 1:
msg = args[0]
args = []
else:
msg = args[0]
args = args[1:]
lines = ['Traceback (most recent call last):\n']
if better:
import better_exchook
better_exchook.better_exchook(*sys.exc_info(),
output=lambda s:lines.append('%s\n' % s))
else:
ei = sys.exc_info()
st = traceback.extract_stack(f=ei[2].tb_frame.f_back)
et = traceback.extract_tb(ei[2])
lines.extend(traceback.format_list(st))
lines.append(' ****** Traceback ****** \n')
lines.extend(traceback.format_list(et))
lines.extend(traceback.format_exception_only(ei[0], ei[1]))
exc = ''.join(lines)
if msg:
args = list(args)
args.append(exc)
logger.error(msg + ':\n%s', *args)
else:
logger.error(exc)
def exchook(exc_type, exc_obj, exc_tb):
"""
Replacement for sys.excepthook.
"""
if exc_type is KeyboardInterrupt:
print("SprintExternInterface[pid %i]: KeyboardInterrupt" % (os.getpid(),))
sys.exit(1)
better_exchook.better_exchook(exc_type, exc_obj, exc_tb)
def extended_exc_hook(*args): better_exchook.better_exchook(*args) for t in threads: async_raise(t.ident, StopMe()) # above doesn't seem to work (yet). # so to be sure: import os os._exit(0)
def exchook(exc_type, exc_obj, exc_tb):
"""
Replacement for sys.excepthook.
"""
if exc_type is KeyboardInterrupt:
print("SprintExternInterface[pid %i]: KeyboardInterrupt" %
(os.getpid(), ))
sys.exit(1)
better_exchook.better_exchook(exc_type, exc_obj, exc_tb)
def items(self): if not CacheEnabled: return from glob import glob for fn in glob(self._filename_pattern()): try: key, value = load(fn) except Exception: print "Exception on loading cache file %s" % fn better_exchook.better_exchook(*sys.exc_info()) raise yield key, value
def run(self):
with self.condition:
if self.wait_time:
self.condition.wait(self.wait_time)
with self.db.lock:
# noinspection PyBroadException
try:
self.do_task()
except Exception:
better_exchook.better_exchook(*sys.exc_info())
finally:
self.db.tasks.remove(self)
def __getitem__(self, key): if not CacheEnabled: raise KeyError try: cache_key, value = load(self._filename_for_key(key)) except IOError: raise KeyError, "cache file not found" except Exception as exc: better_exchook.better_exchook(*sys.exc_info()) if isinstance(exc, KeyError): raise Exception # no KeyError exception fall-through raise else: if cache_key == key: return value raise KeyError, "key repr collidation"
def demo():
"""
Some demo.
"""
# some examples
# this code produces this output: https://gist.github.com/922622
try:
x = {1: 2, "a": "b"}
# noinspection PyMissingOrEmptyDocstring
def f():
y = "foo"
# noinspection PyUnresolvedReferences,PyStatementEffect
x, 42, sys.stdin.__class__, sys.exc_info, y, z
f()
except Exception:
better_exchook(*sys.exc_info())
try:
# noinspection PyArgumentList
(lambda _x: None)(__name__, 42) # multiline
except Exception:
better_exchook(*sys.exc_info())
try:
class Obj:
def __repr__(self):
return ("<Obj multi-\n" + " line repr>")
obj = Obj()
assert not obj
except Exception:
better_exchook(*sys.exc_info())
# noinspection PyMissingOrEmptyDocstring
def f1(a):
f2(a + 1, 2)
# noinspection PyMissingOrEmptyDocstring
def f2(a, b):
f3(a + b)
# noinspection PyMissingOrEmptyDocstring
def f3(a):
b = ("abc" * 100) + "-interesting" # some long demo str
a(b) # error, not callable
try:
f1(13)
except Exception:
better_exchook(*sys.exc_info())
# use this to overwrite the global exception handler
install()
# and fail
# noinspection PyUnresolvedReferences
finalfail(sys)
def excepthook(exc_type, exc_obj, exc_tb):
print "Unhandled exception %s in thread %s, proc %s." % (exc_type, threading.currentThread(), os.getpid())
better_exchook.better_exchook(exc_type, exc_obj, exc_tb)
if main_thread_id == thread.get_ident():
print "We are the main thread."
if not isinstance(exc_type, Exception):
# We are the main thread and we got an exit-exception. This is likely fatal.
# This usually means an exit. (We ignore non-daemon threads and procs here.)
# Print the stack of all other threads.
if hasattr(sys, "_current_frames"):
print ""
threads = {t.ident: t for t in threading.enumerate()}
for tid, stack in sys._current_frames().items():
if tid != main_thread_id:
print "Thread %s:" % threads.get(tid, "unnamed with id %i" % tid)
better_exchook.print_traceback(stack)
print ""
def test_interpreter_helloworld():
testcode = """
#include <stdio.h>
int main(int argc, char** argv) {
printf("Hello %s\n", "world");
printf("args: %i\n", argc);
int i;
for(i = 0; i < argc; ++i)
printf("%s\n", argv[i]);
fflush(stdout);
}
"""
state = helpers_test.parse(testcode, withGlobalIncludeWrappers=True)
from cparser import interpreter
interp = interpreter.Interpreter()
interp.register(state)
def dump():
for f in state.contentlist:
if not isinstance(f, cparser.CFunc): continue
if not f.body: continue
print()
print("parsed content of " + str(f) + ":")
for c in f.body.contentlist:
print(c)
print()
print("PyAST of main:")
interp.dumpFunc("main")
#interpreter.runFunc("main", len(sys.argv), sys.argv + [None])
import os
# os.pipe() returns pipein,pipeout
pipes = os.pipe(), os.pipe() # for stdin/stdout+stderr
if hasattr(os, "set_inheritable"):
# Python 3 by default will close all fds in subprocesses. This will avoid that.
os.set_inheritable(pipes[0][0], True)
os.set_inheritable(pipes[0][1], True)
os.set_inheritable(pipes[1][0], True)
os.set_inheritable(pipes[1][1], True)
pid = os.fork()
if pid == 0: # child
os.close(pipes[0][1])
os.close(pipes[1][0])
os.dup2(pipes[0][0], sys.__stdin__.fileno())
os.dup2(pipes[1][1], sys.__stdout__.fileno())
os.dup2(pipes[1][1], sys.__stderr__.fileno())
try:
interp.runFunc("main", 2, ["./test", "abc", None])
except SystemExit:
raise
except BaseException:
better_exchook.better_exchook(*sys.exc_info())
print("Normal exit.")
os._exit(0)
return
# parent
os.close(pipes[0][0])
os.close(pipes[1][1])
child_stdout = os.fdopen(pipes[1][0], "rb", 0)
child_stdout = child_stdout.readlines()
if PY3:
child_stdout = [l.decode("utf8") for l in child_stdout]
expected_out = [
"Hello world\n",
"args: 2\n",
"./test\n",
"abc\n",
"Normal exit.\n",
]
if expected_out != child_stdout:
print("Got output:")
print("".join(child_stdout))
dump()
print("run directly here now:")
interp.runFunc("main", 2, ["./test", "abc", None])
raise Exception("child stdout %r" % (child_stdout,))
def _debug_shell_exception():
# noinspection PyBroadException
try:
raise Exception("demo exception")
except Exception:
better_exchook(*sys.exc_info(), debugshell=True)
def handle_exc(): print "Exception in thread", curthread.name better_exchook.better_exchook(*excinfo, autodebugshell=False)
def test_interpreter_helloworld():
testcode = """
#include <stdio.h>
int main(int argc, char** argv) {
printf("Hello %s\n", "world");
printf("args: %i\n", argc);
int i;
for(i = 0; i < argc; ++i)
printf("%s\n", argv[i]);
}
"""
state = helpers_test.parse(testcode, withGlobalIncludeWrappers=True)
import interpreter
interp = interpreter.Interpreter()
interp.register(state)
def dump():
for f in state.contentlist:
if not isinstance(f, cparser.CFunc): continue
if not f.body: continue
print
print "parsed content of " + str(f) + ":"
for c in f.body.contentlist:
print c
print
print "PyAST of main:"
interp.dumpFunc("main")
#interpreter.runFunc("main", len(sys.argv), sys.argv + [None])
import os
# os.pipe() returns pipein,pipeout
pipes = os.pipe(), os.pipe() # for stdin/stdout+stderr
if os.fork() == 0: # child
os.close(pipes[0][1])
os.close(pipes[1][0])
os.dup2(pipes[0][0], sys.__stdin__.fileno())
os.dup2(pipes[1][1], sys.__stdout__.fileno())
os.dup2(pipes[1][1], sys.__stderr__.fileno())
try:
interp.runFunc("main", 2, ["./test", "abc", None])
except BaseException:
better_exchook.better_exchook(*sys.exc_info())
os._exit(0)
return
# parent
os.close(pipes[0][0])
os.close(pipes[1][1])
child_stdout = os.fdopen(pipes[1][0])
child_stdout = child_stdout.readlines()
expected_out = [
"Hello world\n",
"args: 2\n",
"./test\n",
"abc\n",
]
if expected_out != child_stdout:
print "Got output:"
print "".join(child_stdout)
dump()
assert False
s1.extend(4, 5) == Stack(0, 1, 2, 3, 4, 5)
return True
if __name__ == "__main__":
# Try to display relevant current values upon any assertion failure.
import os, subprocess
try:
# Use command-line 'pytest' if available to display
# current values upon assertion failure.
subprocess.call(["pytest", "-q", __file__])
except:
try:
# See https://github.com/albertz/py_better_exchook
import sys, traceback, better_exchook
better_exchook.install() # runs: sys.excepthook = better_exchook
except:
# Otherwise, use normal assertion output.
try:
test_stack()
sys.exit()
except:
traceback.print_exc(file=sys.stdout)
raise SystemExit
# better_exchook module installed, use its current assertion values output.
try:
test_stack()
except:
better_exchook.better_exchook(*sys.exc_info())
def test_interpreter_helloworld():
testcode = """
#include <stdio.h>
int main(int argc, char** argv) {
printf("Hello %s\n", "world");
printf("args: %i\n", argc);
int i;
for(i = 0; i < argc; ++i)
printf("%s\n", argv[i]);
}
"""
state = helpers_test.parse(testcode, withGlobalIncludeWrappers=True)
import interpreter
interp = interpreter.Interpreter()
interp.register(state)
def dump():
for f in state.contentlist:
if not isinstance(f, cparser.CFunc): continue
if not f.body: continue
print
print "parsed content of " + str(f) + ":"
for c in f.body.contentlist:
print c
print
print "PyAST of main:"
interp.dumpFunc("main")
#interpreter.runFunc("main", len(sys.argv), sys.argv + [None])
import os
# os.pipe() returns pipein,pipeout
pipes = os.pipe(), os.pipe() # for stdin/stdout+stderr
if os.fork() == 0: # child
os.close(pipes[0][1])
os.close(pipes[1][0])
os.dup2(pipes[0][0], sys.__stdin__.fileno())
os.dup2(pipes[1][1], sys.__stdout__.fileno())
os.dup2(pipes[1][1], sys.__stderr__.fileno())
try:
interp.runFunc("main", 2, ["./test", "abc", None])
except BaseException:
better_exchook.better_exchook(*sys.exc_info())
os._exit(0)
return
# parent
os.close(pipes[0][0])
os.close(pipes[1][1])
child_stdout = os.fdopen(pipes[1][0])
child_stdout = child_stdout.readlines()
expected_out = [
"Hello world\n",
"args: 2\n",
"./test\n",
"abc\n",
]
if expected_out != child_stdout:
print "Got output:"
print "".join(child_stdout)
dump()
assert False
def main():
better_exchook.install()
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument("--ogg")
arg_parser.add_argument("--zip")
arg_parser.add_argument("--ourexec")
arg_parser.add_argument("--libvorbisexec")
arg_parser.add_argument("--ourout")
arg_parser.add_argument("--libvorbisout")
arg_parser.add_argument("--dump_stdout", action="store_true")
arg_parser.add_argument("--no_stderr", action="store_true")
args = arg_parser.parse_args()
if args.no_stderr:
sys.stderr = sys.stdout
if args.zip:
assert not args.ogg and not args.ourout and not args.libvorbisout
sub_cmd = [sys.argv[0]]
if args.ourexec:
sub_cmd += ["--ourexec", args.ourexec]
if args.libvorbisexec:
sub_cmd += ["--libvorbisexec", args.libvorbisexec]
if args.dump_stdout:
sub_cmd += ["--dump_stdout"]
if args.no_stderr:
sub_cmd += ["--no_stderr"]
import zipfile
ogg_count = 0
with zipfile.ZipFile(args.zip) as zip_f:
for fn in zip_f.namelist():
print(fn)
if fn.endswith(".ogg"):
ogg_count += 1
ogg_raw_bytes = zip_f.read(fn)
with tempfile.NamedTemporaryFile(
suffix=os.path.basename(fn)) as tmp_f:
tmp_f.write(ogg_raw_bytes)
tmp_f.flush()
call(sub_cmd + ["--ogg", tmp_f.name])
print("Found %i OGG files." % ogg_count)
return
if args.ogg:
assert not args.ourout, "--ogg xor --ourout.\n%s" % arg_parser.format_usage(
)
if args.ourexec is None:
assert os.path.exists(
default_ours_exec), "run `compile-libvorbis.py --mode ours`"
args.ourexec = default_ours_exec
args.ourout = create_debug_out(args.ourexec, args.ogg)
if args.libvorbisexec is None:
assert os.path.exists(
default_libvorbis_exec
), "run `compile-libvorbis.py --mode standalone`"
args.libvorbisexec = default_libvorbis_exec
if args.libvorbisexec:
assert not args.libvorbisout
args.libvorbisout = create_debug_out(args.libvorbisexec, args.ogg)
assert args.ourout, "need --ourout.\n%s" % arg_parser.format_usage()
reader1 = Reader(args.ourout)
print("Read our (ParseOggVorbis) debug out file:", reader1.filename)
print("Our decoder name:", reader1.decoder_name)
print("Our num channels:", reader1.decoder_num_channels)
print("Our sample rate:", reader1.decoder_sample_rate)
reader2 = None
if args.libvorbisout:
reader2 = Reader(args.libvorbisout)
print("Read libvorbis debug out file:", reader2.filename)
print("libvorbis decoder name:", reader2.decoder_name)
print("libvorbis num channels:", reader2.decoder_num_channels)
print("libvorbis sample rate:", reader2.decoder_sample_rate)
print("Will check that both are the same.")
assert reader2.decoder_sample_rate == reader1.decoder_sample_rate
assert reader2.decoder_num_channels == reader1.decoder_num_channels
reader1.read_setup(dump=args.dump_stdout)
if reader2:
reader2.read_setup(dump=args.dump_stdout)
assert len(reader1.floors) == len(reader2.floors)
for f1, f2 in zip(reader1.floors, reader2.floors):
assert f1.multiplier == f2.multiplier
assert len(f1.xs) == len(f2.xs)
for x1, x2 in zip(f1.xs, f2.xs):
assert x1 == x2
num_packets = 0
while True:
packet1 = reader1.read_audio_packet(dump=args.dump_stdout)
if reader2:
packet2 = reader2.read_audio_packet(dump=args.dump_stdout)
try:
AudioPacket.assert_same(packet1, packet2)
assert sorted(reader1.pcm_data.keys()) == sorted(
reader2.pcm_data.keys())
for channel in sorted(reader1.pcm_data.keys()):
pcms1 = reader1.pcm_data[channel]
pcms2 = reader2.pcm_data[channel]
assert len(pcms1) == len(pcms2)
pcm1 = sum(pcms1, tuple())
pcm2 = sum(pcms2, tuple())
min_len = min(len(pcm1), len(pcm2))
assert_close_list(pcm1[:min_len], pcm2[:min_len])
pcms1.clear()
pcms2.clear()
if len(pcm1) > min_len:
pcms1.append(pcm1)
if len(pcm2) > min_len:
pcms2.append(pcm2)
if not pcms1:
del reader1.pcm_data[channel]
if not pcms2:
del reader2.pcm_data[channel]
except Exception:
print(
"Exception at packet %i, num samples1 %r, num samples2 %r (diff is valid here)."
% (num_packets, reader1.num_samples, reader2.num_samples))
try:
num_remaining_packets1 = reader1.count_remaining_audio_packets(
dump=args.dump_stdout)
num_remaining_packets2 = reader2.count_remaining_audio_packets(
dump=args.dump_stdout)
print("Num remaining reader1: %i, reader2: %i" %
(num_remaining_packets1, num_remaining_packets2))
except Exception:
print(
"During count_remaining_audio_packets, another exception occured:"
)
better_exchook.better_exchook(*sys.exc_info())
print("Reraising original exception now.")
raise
if packet1.eof:
print("EOF")
if reader2:
assert not reader1.pcm_data and not reader2.pcm_data
break
num_packets += 1
print("Finished.")
print("Num audio packets:", num_packets)
print("Reader1 num samples:", reader1.num_samples)
if reader2:
print("Reader2 num samples:", reader2.num_samples)
