def test_clear(self):
def outer():
middle()
def middle():
inner()
def inner():
i = 1
1 / 0
try:
outer()
except:
type_, value, tb = sys.exc_info()
# Initial assertion: there's one local in the inner frame.
inner_frame = tb.tb_next.tb_next.tb_next.tb_frame
self.assertEqual(len(inner_frame.f_locals), 1)
# Clear traceback frames
traceback.clear_frames(tb)
# Local variable dict should now be empty.
self.assertEqual(len(inner_frame.f_locals), 0)
def tworker():
global _rspq
while True:
try:
msg = _reqq.get()
except AttributeError:
return # can't get anything off of a dud queue
if msg is None:
return
(e, meth, args, kwargs) = msg
rv = None
try:
rv = meth(*args, **kwargs)
except SYS_EXCS:
raise
except EXC_CLASSES:
rv = sys.exc_info()
if sys.version_info >= (3, 4):
traceback.clear_frames(rv[1].__traceback__)
if six.PY2:
sys.exc_clear()
# test_leakage_from_tracebacks verifies that the use of
# exc_info does not lead to memory leaks
_rspq.put((e, rv))
msg = meth = args = kwargs = e = rv = None
_wsock.sendall(_bytetosend)
def erase_traceback(exc_value):
"""
Erase the traceback and hanging locals from the given exception instance.
"""
if exc_value.__traceback__ is not None:
traceback.clear_frames(exc_value.__traceback__)
return exc_value.with_traceback(None)
def _clear_unnecessary_locals():
# For Stackless Python 3.6
if hasattr(traceback_lib, "clear_frames"):
traceback = sys.exc_info()[2]
if traceback:
traceback_lib.clear_frames(traceback)
return
# For Stackless Python 2.7
frame = inspect.currentframe().f_back
target_frames = []
try:
while frame:
locals_hash = frame.f_locals
if locals_hash and "_tag_for_clear_unnecessary_locals" in locals_hash:
break
target_frames.append(frame)
frame = frame.f_back
traceback = sys.exc_info()[2]
try:
if traceback:
while traceback:
_clear_locals_in_traceback(traceback, target_frames)
traceback = traceback.tb_next
for frame in target_frames:
traceback = _find_traceback_in_frame(frame)
while traceback:
_clear_locals_in_traceback(traceback, target_frames)
traceback = traceback.tb_next
finally:
del traceback
finally:
del frame
del target_frames
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except TaskState.UserCancelException as e:
# TODO: Is this enough to allow immediate gc of the stack?
# How does it chain across cython code?
# Maybe just return None.
e = e.with_traceback(None)
e.__context__ = None
e.__cause__ = None
raise e
except BaseException as e:
traceback.clear_frames(e.__traceback__)
raise
def test_clear(self):
def outer():
middle()
def middle():
inner()
def inner():
i = 1
1/0
try:
outer()
except:
type_, value, tb = sys.exc_info()
# Initial assertion: there's one local in the inner frame.
inner_frame = tb.tb_next.tb_next.tb_next.tb_frame
self.assertEqual(len(inner_frame.f_locals), 1)
# Clear traceback frames
traceback.clear_frames(tb)
# Local variable dict should now be empty.
self.assertEqual(len(inner_frame.f_locals), 0)
def handle(target, plugin, name):
target_uid = target.get('uid')
target_content = target.get('target')
plugin_name = plugin.get('name')
plugin_option = plugin.get('option')
try:
result = getattr(scanner.plugin, plugin_name)(target_content,
plugin_option)
except Exception as e:
exc, tb = sys.exc_info()[1:]
result = str(exc.__class__, exc, ''.join(traceback.format_tb(tb)))
traceback.clear_frames(tb)
del tb
es.index(index='node-{}-{}'.format(
name, time.strftime('%Y-%m-%d', time.localtime())),
doc_type='doc',
body={
'target': target,
'plugin': plugin,
'result': result,
'timestamp': datetime.utcnow()
})
return {'uid': target_uid, 'result': result}
async def proxy(self, host, port, request_type, data, client_reader, client_writer, type):
server_writer = None
try:
if type:
if self.connection_pool == []:
server_reader, server_writer = await asyncio.open_connection(host=self.config['host'],
port=self.config['port'],
ssl=self.context,
server_hostname=self.config['host'])
server_writer.write(self.config['uuid'])
await server_writer.drain()
else:
server_reader, server_writer = self.connection_pool.pop(0)
server_writer.write(int.to_bytes(len(host + b'\n' + port + b'\n'), 2, 'big', signed=True))
await server_writer.drain()
server_writer.write(host + b'\n' + port + b'\n')
await server_writer.drain()
else:
address = (await self.loop.getaddrinfo(host=host, port=port, family=0, type=socket.SOCK_STREAM))[0][4]
if address[0] != '127.0.0.1':
server_reader, server_writer = await asyncio.open_connection(host=address[0], port=address[1])
else:
if not request_type:
client_writer.write(b'''HTTP/1.1 404 Not Found\r\nProxy-Connection: close\r\n\r\n''')
await client_writer.drain()
raise Exception
if not request_type:
client_writer.write(b'''HTTP/1.1 200 Connection Established\r\nProxy-Connection: close\r\n\r\n''')
await client_writer.drain()
elif data != None:
server_writer.write(data)
await server_writer.drain()
return server_reader, server_writer
except Exception as e:
traceback.clear_frames(e.__traceback__)
e.__traceback__ = None
await self.clean_up(client_writer, server_writer)
def exception_to_traceback(exception, file: str) -> str:
import traceback
traceback.clear_frames(exception.__traceback__)
tb_summary_list = list(traceback.extract_tb(exception.__traceback__))
# Remove stack frames from the stack trace that come from the repl implementation.
for line_number, tb_summary in enumerate(tb_summary_list):
if tb_summary.filename == file:
tb_summary_list = tb_summary_list[line_number:]
break
else:
# Leave traceback unaltered.
pass
# raise Exception("What should happen here??")
if hasattr(exception, "extra_traceback"):
tb_summary_list.extend(exception.extra_traceback)
l = traceback.format_list(tb_summary_list)
if l:
l.insert(0, "Traceback (most recent call last):\n")
l.extend(traceback.format_exception_only(type(exception), exception))
return "".join(l)
async def switch(self, reader, writer, other, up):
try:
if not up:
while True:
data = await reader.read(16384)
writer.write(data)
await writer.drain()
if data == b'':
break
else:
while True:
data = await reader.read(16384)
if data[:3] == b'GET' or data[:4] == b'POST':
URL, host, port = self.get_http_address(data, 1)
data = self.get_response(data, host, port)
writer.write(data)
await writer.drain()
if data == b'':
break
await self.clean_up(writer, other)
except Exception as e:
traceback.clear_frames(e.__traceback__)
e.__traceback__ = None
await self.clean_up(writer, other)
def evaluate_code(code, arguments, tests, executor_):
stats = {
'tests-executed': len(tests),
'tests-passed': 0,
'result-none': 0,
'syntax-error': 0,
'runtime-exception': 0,
'exceptions': []
}
if not code:
return stats
for test in tests:
try:
execution_result = executor_.execute(code, arguments,
test['input'])
except EvaluationTimeout:
raise EvaluationTimeout()
except ExecutorSyntaxException as e:
traceback.clear_frames(e.__traceback__)
stats['syntax-error'] += 1
stats['exceptions'].append(str(e))
continue
except ExecutorRuntimeException as e:
traceback.clear_frames(e.__traceback__)
stats['runtime-exception'] += 1
stats['exceptions'].append(str(e))
continue
except Exception as e:
traceback.clear_frames(e.__traceback__)
#print("Exception: %s" % e)
#traceback.print_exc()
#print(code, arguments, test['input'])
stats['exceptions'].append(str(e))
continue
if execution_result.result is None:
stats['result-none'] += 1
if executor_.compare(test['output'], execution_result.result):
stats['tests-passed'] += 1
return stats
def __exit__(self, exc_type, exc_val, exc_tb):
if not exc_val: return True
traceback.clear_frames(exc_tb)
gc.collect()
raise exc_type(exc_val).with_traceback(exc_tb) from None
def get_ref_free_exc_info():
"Free traceback from references to locals() in each frame to avoid circular reference leading to gc.collect() unable to reclaim memory"
type, val, tb = sys.exc_info()
traceback.clear_frames(tb)
return (type, val, tb)
def run_exp(self,
exp_function: list or function,
params: dict = None,
artifacts: dict = None):
"""
Runs the given experiment function or list of functions and updates the experiment metadata
# Arguments
exp_function (function or list): Method that implements the experiment.
params (dict): Dictionary that contains the configuration (e.g. hyperparameters) for the experiment (optional).
artifacts (dict): Dictionary that contains artifacts (any kind of python object) required for the experiment (optional).
"""
# TODO track file events only during run?
if self._running:
self.log.warning(
"Experiment is already running. Running same experiment in parallel "
"might give some trouble.")
elif self._has_run:
self.log.info(
"This experiment has already been run. Metadata will be overwritten! "
"It is suggested to initialize a new experiment. "
"The metadata of the last run is still saved in a run.json in the local exp folder."
)
# Redirect stdout/sterr to file
if self._stdout_file_redirect is None:
self._stdout_file_redirect = experiment_utils.StdoutFileRedirect(
log_path=self.stdout_path)
if self.redirect_logs:
self._stdout_file_redirect.redirect()
# Executing experiment functions
self.log.info("Running experiment: " + self.name)
self._running = True
self._has_run = True # even
self._syncing = False
# add artifacts
if artifacts is not None:
for artifact_key in artifacts:
self.add_artifact(artifact_key, artifacts[artifact_key])
# Log params
if params is None:
params = {}
else:
params = params.copy()
self.log_params(params)
# Wraps the experiment function into another function for more control
def exp_wrapper():
result_value = None
kwargs = {
"exp": self,
"params": self.params,
"artifacts": self._artifacts,
"config": self.params,
"log": self.log
}
if type(exp_function) is list:
for exp_func in exp_function:
if not callable(exp_func):
self.log.warning(str(exp_func) + " is not a function.")
continue
self.log.info("Running experiment function: " +
exp_func.__name__)
if not self.exp_metadata.command:
self.exp_metadata.command = exp_func.__name__
else:
self.exp_metadata.command += " -> " + exp_func.__name__
result_value = call_function(exp_func, **kwargs)
elif callable(exp_function):
self.log.info("Running experiment function: " +
exp_function.__name__)
self.exp_metadata.command = exp_function.__name__
result_value = call_function(exp_function, **kwargs)
else:
self.log.error(str(exp_function) + " is not a function.")
if result_value:
return result_value
sync_heartbeat = None
heartbeat_stop_event = None
if self.auto_sync:
# only required for auto sync
# sync hearbeat for every 20 seconds - make configurable?
# run sync exp without uploading resources
heartbeat_stop_event, sync_heartbeat = IntervalTimer.create(
self.sync_exp, 20)
# Initial sync of metadata
self.sync_exp(upload_resources=False)
try:
if sync_heartbeat:
# Start heartbeat if initialized
sync_heartbeat.start()
self.exp_metadata.status = ExperimentState.RUNNING
self.set_completed(exp_wrapper())
except:
ex_type, val, tb = sys.exc_info()
if ex_type is KeyboardInterrupt:
# KeyboardInterrupt cannot be catched via except Exception: https://stackoverflow.com/questions/4990718/about-catching-any-exception
self.exp_metadata.status = ExperimentState.INTERRUPTED
else:
self.exp_metadata.status = ExperimentState.FAILED
self._finish_exp_run()
# TODO: Move this part into utils?
if notebook_utils.in_ipython_environment() and len(
self.exp_metadata.host.gpus) > 0:
# TODO only call on keyboard interrupt and CUDA problems? debug modus will not be available
# if in jupyter and gpus available -> clear stack trace to free GPU memory
# https://docs.fast.ai/troubleshoot.html#custom-solutions
# https://github.com/fastai/fastai/blob/master/fastai/utils/ipython.py
traceback.clear_frames(tb)
# Collect Garbage
gc.collect()
# https://github.com/stas00/ipyexperiments/blob/master/ipyexperiments/ipyexperiments.py
# now we can attempt to reclaim GPU memory - not needed?
# try:
# import torch
# torch.cuda.empty_cache()
# except:
# pass
raise ex_type(val).with_traceback(tb) from None
raise
finally:
if sync_heartbeat:
# Always stop heartbeat if initialized
heartbeat_stop_event.set()
sync_heartbeat.join(timeout=2)
def update_event(self, inp=-1):
self.set_output_val(0, traceback.clear_frames(self.input(0)))
def call_unit():
try:
use_cache = True
if arguments_pass_type in [
ARGS_COMPILE_ONLY, ARGS_PASS_LISP
]:
arg_values = values
use_cache = False
else:
arg_values = [
value() if call else value
for value, call in values
]
if func == 'dict-new':
use_cache = False
if trace is not None:
trace_id = trace.add_call(func, arg_values)
if use_cache:
try:
cache_key1 = lists_to_tuples(statement)
except EvaluationTimeout:
raise EvaluationTimeout()
except Exception as e:
use_cache = False
try:
cache_key2 = lists_to_tuples(arg_values)
except EvaluationTimeout:
raise EvaluationTimeout()
except Exception as e:
use_cache = False
try:
cache_key3 = lists_to_tuples(argument_map)
except EvaluationTimeout:
raise EvaluationTimeout()
except Exception as e:
use_cache = False
if use_cache:
cache_key = (cache_key1, cache_key2, cache_key3)
if cache_key in _EXECUTION_CACHE:
ret = _EXECUTION_CACHE[cache_key]
if trace is not None:
trace.add_result(trace_id, ret)
return ret
if argument_map and func in argument_map:
ret = context[func](*arg_values)
else:
if arguments_pass_type == ARGS_COMPILE_ONLY:
ret = unit.compute(context, *arg_values)
elif arguments_pass_type == ARGS_PASS_LISP:
ret = unit.compute(argument_map, context,
*arg_values)
else:
ret = unit.compute(*arg_values)
if use_cache:
_EXECUTION_CACHE[cache_key] = ret
if trace is not None:
trace.add_result(trace_id, ret)
return ret
except TypeError as e:
traceback.clear_frames(e.__traceback__)
# print("TypeError %s calling %s, arguments: %s. Program: %s" % (
# str(e), func, values, statement))
raise
def evaluate_datum(i,
datum,
model,
dcModel,
nRepeats,
mdl,
max_to_check,
timeout=None):
try:
if timeout is not None:
#print("hit set timeout")
def timeoutCallBack(_1, _2):
raise EvaluationTimeout()
#signal.signal(signal.SIGVTALRM, timeoutCallBack)
#signal.setitimer(signal.ITIMER_REAL, timeout)
signal.signal(signal.SIGALRM, timeoutCallBack)
signal.alarm(timeout)
t = time.time()
results = []
print("eval datum number", i)
samples = {('<HOLE>', )} # make more general # TODO, i don't think
n_checked, n_hit = 0, 0
if model:
#print("num threads:", torch.get_num_threads())
torch.set_num_threads(1)
tnet = time.time()
#print("task", i, "started using rnn")
spec = tokenize_for_robustfill([
datum.spec
]) if not args.IO2seq else tokenize_IO_for_robustfill(
[datum.IO], digit_enc=args.digit_enc)
if args.beam:
samples, _scores = model.beam_decode(spec, beam_size=nRepeats)
else:
samples, _scores, _ = model.sampleAndScore(spec,
nRepeats=nRepeats)
#print("task", i, "done with rnn, took", time.time()-t, "seconds", flush=True)
# only loop over unique samples:
samples = {tuple(sample) for sample in samples} # only
if (not improved_dc_grammar) or (not dcModel):
g = basegrammar if not dcModel else dcModel.infer_grammar(
(datum.spec if not args.IO2seq else datum.IO)) # TODO pp
g = untorch(g)
sketchtups = []
for sample in samples:
try:
tr = seq_to_tree(sample)
#print(tr)
sk = tree_to_prog(tr)
except EvaluationTimeout:
raise EvaluationTimeout()
except Exception as e: # TODO: needs to be fixed
traceback.clear_frames(e.__traceback__)
print("EXCEPTION IN PARSE:,", e)
#traceback.print_tb(e.__traceback__)
del e
n_checked += 1
results.append(
AlgolispResult(sample, None, False, n_checked,
time.time() - t))
continue
if improved_dc_grammar:
g = untorch(
dcModel.infer_grammar(
((datum.spec if not args.IO2seq else datum.IO),
sample)))
sketchtups.append(SketchTup(sk, g))
# only loop over unique sketches:
sketchtups = {sk for sk in sketchtups} #fine
#print("sketchtups:", sketchtups)
#alternate which sketch to enumerate from each time
#print("task", i, "starting enum outer, took", time.time()-t, "seconds", flush=True)
if args.pypy:
enum_results, n_checked, n_hit = pypy_enumerate(
datum.tp, datum.IO, datum.schema_args, mdl, sketchtups,
n_checked, n_hit, t, max_to_check, i)
else:
enum_results, n_checked, n_hit = algolisp_enumerate(
datum.tp, datum.IO, datum.schema_args, mdl, sketchtups,
n_checked, n_hit, t, max_to_check, i) #might need more than IO
del sketchtups
#del g
if model:
del model
if dcModel:
del dcModel
return results + enum_results
except KeyError as e:
print("on task", i, "KeyError:", e)
return [AlgolispResult(None, None, False, n_checked, time.time() - t)]
except EvaluationTimeout:
print("Timed out while evaluating task", i)
return [AlgolispResult(None, None, False, n_checked, time.time() - t)]
finally:
######TODO: want search time and total time to hit task ######
if timeout is not None:
#signal.signal(signal.SIGVTALRM, lambda *_: None)
#signal.setitimer(signal.ITIMER_REAL, 0)
signal.alarm(0)
print(f"task {i}:")
print(f"evaluation for task {i} took {time.time()-t} seconds")
print(
f"For task {i}, tried {n_checked} candidates, found {n_hit} hits",
flush=True)
def test_lisp_validity(lisp_units,
short_tree,
args_map,
root_type,
constants={},
name=None):
#print(short_tree, root_type)
if isinstance(short_tree, six.string_types):
if short_tree in lisp_units:
x = lisp_units[short_tree]
r = x.return_type
if x.args:
if not isinstance(root_type, FuncType):
raise TypeError("Using function %s when %s expected." %
(x, root_type))
if any_type(root_type.return_type
) and not root_type.return_type.compatible(
root_type.return_type):
raise TypeError(
"Function %s return type %s is not compatible with expected %s return type %s"
% (x, x.return_type, root_type, root_type.return_type))
if len(x.args) != len(root_type.argument_types):
raise TypeError(
"Function %s number of arguments %s doesn't match %s."
% (x, x.args, root_type))
args = []
for i, arg in enumerate(x.args):
if callable(arg):
tp = arg(x.return_type, root_type.argument_types[:i])
else:
tp = arg
if not any_type(
root_type.argument_types[i]
) and not root_type.argument_types[i].compatible(tp):
raise TypeError(
"Passing function %s(%s) when %s expected. Argument %d mismatch: %s vs %s."
% (x, [arg for arg in x.args], root_type, i, tp,
root_type.argument_types[i]))
#FuncType(x.return_type, [arg for arg in root_type.args])
r = FuncType(
x.return_type if not any_type(x.return_type) else
root_type.return_type, [
arg if not callable(arg) else root_arg for arg,
root_arg in zip(x.args, root_type.argument_types)
])
# ????
# if x.computed_return_type is not None:
# print(x, x.computed_return_type, r, root_type)
# r = functools.partial(x.computed_return_type, root_type)
#print(x.name, r, x.computed_return_type, root_type)
elif short_tree in constants:
x = constants[short_tree]
r = x.get_type()
elif short_tree in args_map:
r = args_map[short_tree]
else:
raise ValueError("Unknown symbol %s. Current args: %s" %
(short_tree, args_map))
if not r.compatible(root_type):
raise TypeError(
"Symbol %s with return type %s is not compatible with "
"expected type %s" % (short_tree, r, root_type))
else:
if short_tree[0] in constants and short_tree[0] not in mapping:
# TODO: validate built-in functions
return
if short_tree[0] in args_map:
r = args_map[short_tree[0]]
return r
x = lisp_units[short_tree[0]]
r = x.return_type
args = []
if not r.compatible(root_type):
raise TypeError(
"Return type %s of function %s doesn't match with expected return type %s: %s"
% (r, short_tree[0], root_type, short_tree))
assert r.compatible(root_type), (short_tree, r, root_type)
if len(x.args) + 1 != len(short_tree):
raise ValueError("Wrong number of arguments in %s, expected: %s" %
(short_tree, x.args))
assert len(x.args) + 1 == len(short_tree), short_tree
# Handling lambdas.
if x.name == 'lambda1':
assert len(short_tree) == 2
new_args_map = copy.copy(args_map)
new_args_map['arg1'] = Number.T
new_args_map['self'] = FuncType(root_type, [Number.T])
ret = test_lisp_validity(lisp_units, short_tree[1], new_args_map,
root_type.return_type, constants)
r = FuncType(ret, [Number.T])
elif x.name == 'lambda2':
assert len(short_tree) == 2
new_args_map = copy.copy(args_map)
new_args_map['arg1'] = Number.T
new_args_map['arg2'] = Number.T
new_args_map['self'] = FuncType(root_type, [Number.T, Number.T])
ret = test_lisp_validity(lisp_units, short_tree[1], new_args_map,
root_type.return_type, constants)
r = FuncType(ret, [Number.T, Number.T])
else:
# Regular function call.
for arg, sub in zip(x.args, short_tree[1:]):
if callable(arg):
try:
tp = arg(root_type, args)
except EvaluationTimeout:
raise EvaluationTimeout()
except:
traceback.clear_frames(e.__traceback__)
# print("Error while executing: %s, arg: %s, root_type: %s, args: %s" % (
# short_tree, arg, root_type, args))
raise
else:
tp = arg
# print(sub, arg, tp, root_type, args, lisp_units[short_tree[0]].args)
args.append(
test_lisp_validity(lisp_units, sub, args_map, tp,
constants))
if args[-1].__class__ == Type:
raise TypeError("Argument %s resolved to %s." %
(sub, args[-1]))
if x.computed_return_type is not None:
r = x.computed_return_type(root_type, args)
assert r.compatible(
root_type
), "Computed type for %s -> %s doesn't match expected type %s. Probably issue with argument computation." % (
short_tree, r, root_type)
# print(short_tree, r)
return r
