def start(self, parallel_mode=False):
self.get_ready()
if self.nesting == 0: #pragma: no cover
sys.settrace(self.t)
if hasattr(threading, 'settrace'):
threading.settrace(self.t)
self.nesting += 1
def start(self):
assert not self.started, "can't start if already started"
if not self.donothing:
sys.settrace = settrace
sys.settrace(self.globaltrace)
threading.settrace(self.globaltrace)
self.started = True
def start(self):
self.get_ready()
if self.nesting == 0: # pragma: no cover
sys.settrace(self.t)
if hasattr(threading, "settrace"):
threading.settrace(self.t)
self.nesting += 1
def stop(self):
if self._handler is not None:
sys.settrace(self._previous)
self._handler = self._previous = None
if self.threading_support:
threading.settrace(self._threading_previous)
self._threading_previous = None
def start_instrumenting(output_file, to_include=(), to_exclude=()): """Enable tracing of all function calls (from specified modules).""" trace_event.trace_enable(output_file) traceFunc = _generate_trace_function(to_include, to_exclude) sys.settrace(traceFunc) threading.settrace(traceFunc)
def start(self):
"""Start collecting trace information."""
if self._collectors:
self._collectors[-1].pause()
self._collectors.append(self)
#print >>sys.stderr, "Started: %r" % self._collectors
# Check to see whether we had a fullcoverage tracer installed.
traces0 = None
if hasattr(sys, "gettrace"):
fn0 = sys.gettrace()
if fn0:
tracer0 = getattr(fn0, '__self__', None)
if tracer0:
traces0 = getattr(tracer0, 'traces', None)
# Install the tracer on this thread.
fn = self._start_tracer()
if traces0:
for args in traces0:
(frame, event, arg), lineno = args
fn(frame, event, arg, lineno=lineno)
# Install our installation tracer in threading, to jump start other
# threads.
threading.settrace(self._installation_trace)
def run(args):
try:
import threading
main_thread = threading.current_thread().ident
#logging.info("Starting Now")
#print sys.argv
config = ServerOptions()
config.parseOptions()
#app.runReactorWithLogging(
# config,
# sys.stdout,
# sys.stderr,
# TwistedFlameGraphProfiler(config['profile'], False),
# None
#)
#runner = app.ApplicationRunner(config)
#runner.profiler = TwistedFlameGraphProfiler("/tmp/twisted_flamegraph.log", False)
#runner.startReactor(None, sys.stdout, sys.stderr )
#runner.run()
sys.settrace(trace)
threading.settrace(trace)
UnixApplicationRunner(config).run()
#app.run(args[0], ServerOptions)
except Exception as e:
print e.message
finally:
#while threading.active_count() > 1:
# time.sleep(0.1)
sys.settrace(None)
threading.settrace(None)
write_stats()
def start(filter_modules=DEFAULT_MODULES):
tracer.init()
if filter_modules is not None:
tracer.set_filter_modules(filter_modules)
threading.settrace(thread_trace)
tracer.start_dumper()
tracer.install_hook()
def run(self, file, globals=None, locals=None):
if globals is None:
#patch provided by: Scott Schlesier - when script is run, it does not
#use globals from pydevd:
#This will prevent the pydevd script from contaminating the namespace for the script to be debugged
#pretend pydevd is not the main module, and
#convince the file to be debugged that it was loaded as main
sys.modules['pydevd'] = sys.modules['__main__']
sys.modules['pydevd'].__name__ = 'pydevd'
from imp import new_module
m = new_module('__main__')
sys.modules['__main__'] = m
m.__file__ = file
globals = m.__dict__
if locals is None:
locals = globals
#Predefined (writable) attributes: __name__ is the module's name;
#__doc__ is the module's documentation string, or None if unavailable;
#__file__ is the pathname of the file from which the module was loaded,
#if it was loaded from a file. The __file__ attribute is not present for
#C modules that are statically linked into the interpreter; for extension modules
#loaded dynamically from a shared library, it is the pathname of the shared library file.
#I think this is an ugly hack, bug it works (seems to) for the bug that says that sys.path should be the same in
#debug and run.
if m.__file__.startswith(sys.path[0]):
#print >> sys.stderr, 'Deleting: ', sys.path[0]
del sys.path[0]
#now, the local directory has to be added to the pythonpath
#sys.path.insert(0, os.getcwd())
#Changed: it's not the local directory, but the directory of the file launched
#The file being run ust be in the pythonpath (even if it was not before)
sys.path.insert(0, os.path.split(file)[0])
# for completness, we'll register the pydevd.reader & pydevd.writer threads
net = NetCommand(str(CMD_THREAD_CREATE), 0, '<xml><thread name="pydevd.reader" id="-1"/></xml>')
self.writer.addCommand(net)
net = NetCommand(str(CMD_THREAD_CREATE), 0, '<xml><thread name="pydevd.writer" id="-1"/></xml>')
self.writer.addCommand(net)
pydevd_tracing.SetTrace(self.trace_dispatch)
try:
#not available in jython!
threading.settrace(self.trace_dispatch) # for all future threads
except:
pass
while not self.readyToRun:
time.sleep(0.1) # busy wait until we receive run command
PyDBCommandThread(debugger).start()
execfile(file, globals, locals) #execute the script
async def disable_tracing(self):
if not config.get("tracing.enabled", False):
return
self.log_data["message"] = "disabling tracing"
log.debug(self.log_data)
sys.settrace(None)
threading.settrace(None)
def tearDownClass(cls) -> None:
# reset to the default tracing function
threading.settrace(cls.__default_trace)
# reset the singleton classes (for further tests)
TSingleton._instance = None
TSingletonArgs._instances.clear() # noqa
def start(self, parallel_mode=False):
self.get_ready()
if self.nesting == 0: #pragma: no cover
sys.settrace(self.t)
if hasattr(threading, 'settrace'):
threading.settrace(self.t)
self.nesting += 1
def start(self):
"""Start collecting trace information."""
if self._collectors:
self._collectors[-1].pause()
self._collectors.append(self)
#print >>sys.stderr, "Started: %r" % self._collectors
# Check to see whether we had a fullcoverage tracer installed.
traces0 = None
if hasattr(sys, "gettrace"):
fn0 = sys.gettrace()
if fn0:
tracer0 = getattr(fn0, '__self__', None)
if tracer0:
traces0 = getattr(tracer0, 'traces', None)
# Install the tracer on this thread.
fn = self._start_tracer()
if traces0:
for args in traces0:
(frame, event, arg), lineno = args
fn(frame, event, arg, lineno=lineno)
# Install our installation tracer in threading, to jump start other
# threads.
threading.settrace(self._installation_trace)
def thread_wrapper(self):
secs = getattr(test_func, '__timeout__', 0) or \
getattr(self.__class__, '__timeout__', 0) or \
_TimeoutThread.timeout
if secs <= 0:
return test_func(self)
try:
thread = _TimeoutThread(name=test_func.__qualname__,
target=test_func, args=(self,))
threading.settrace(thread.global_trace)
thread.start()
thread.join(secs)
alive = thread.isAlive()
thread.kill()
thread.join()
finally:
threading.settrace(None)
if thread.exc_info is not None:
raise thread.exc_info[1].with_traceback(thread.exc_info[2])
if alive:
raise unittest.SkipTest(f'Function ran longer than {secs} second(s)')
return None
def stop(self):
assert self.started, "can't stop if not started"
if not self.donothing:
sys.settrace = osettrace
sys.settrace(None)
threading.settrace(None)
self.started = False
def start_instrumenting(output_file, to_include=(), to_exclude=()):
"""Enable tracing of all function calls (from specified modules)."""
trace_event.trace_enable(output_file)
traceFunc = _generate_trace_function(to_include, to_exclude)
sys.settrace(traceFunc)
threading.settrace(traceFunc)
def log(log_path, module_names=[], sub=False, all=False):
# For well formed lines in multithreaded programs:
writelock = threading.Lock()
set_file(log_path)
def write(module_name, lineno, function, line):
timestamp = datetime.now().strftime(
"%Y-%m-%d %H:%M:%S.%f")[:-3] # chop microseconds to milliseconds
threadname = threading.current_thread().name
message = "[%s] %s: %s:%s in %s: %s\n" % (
timestamp, threadname, module_name, lineno, function, line)
with writelock:
outfile.write(message)
def traceit(frame, event, arg):
if event == "line":
filename, lineno, function, code_context, index = inspect.getframeinfo(
frame, context=1)
try:
module_name = frame.f_globals["__name__"]
except KeyError:
module_name = '<string>'
if module_name in module_names or all or (sub and any(
[module_name.startswith(s) for s in module_names])):
line = code_context[0].rstrip(
) if code_context else '<within exec() or eval()>'
write(module_name, lineno, function, line)
return traceit
write('tracelog', '', '', '\n\n***starting***\n')
threading.settrace(traceit)
sys.settrace(traceit)
def stop(self):
assert self.started, "can't stop if not started"
if not self.donothing:
sys.settrace = osettrace
sys.settrace(None)
threading.settrace(None)
self.started = False
def start(filter_modules=DEFAULT_MODULES):
tracer.init()
if filter_modules is not None:
tracer.set_filter_modules(filter_modules)
threading.settrace(thread_trace)
tracer.start_dumper()
tracer.install_hook()
def trace(self, predicate):
"""
Starts tracing with the given callable.
Args:
predicate (callable that accepts a single :obj:`~hunter.event.Event` argument):
Return:
self
"""
self._handler = predicate
if self.profiling_mode:
if self.threading_support is None or self.threading_support:
self._threading_previous = getattr(threading, '_profile_hook',
None)
threading.setprofile(self)
self._previous = sys.getprofile()
sys.setprofile(self)
else:
if self.threading_support is None or self.threading_support:
self._threading_previous = getattr(threading, '_trace_hook',
None)
threading.settrace(self)
self._previous = sys.gettrace()
sys.settrace(self)
return self
def start(self):
"""Start collecting trace information."""
if self._collectors:
self._collectors[-1].pause()
self._collectors.append(self)
#print("Started: %r" % self._collectors, file=sys.stderr)
# Check to see whether we had a fullcoverage tracer installed.
traces0 = []
if hasattr(sys, "gettrace"):
fn0 = sys.gettrace()
if fn0:
tracer0 = getattr(fn0, '__self__', None)
if tracer0:
traces0 = getattr(tracer0, 'traces', [])
# Install the tracer on this thread.
fn = self._start_tracer()
for args in traces0:
(frame, event, arg), lineno = args
try:
fn(frame, event, arg, lineno=lineno)
except TypeError:
raise Exception(
"fullcoverage must be run with the C trace function."
)
# Install our installation tracer in threading, to jump start other
# threads.
threading.settrace(self._installation_trace)
def pause(self):
"""
Pause tracing, but be prepared to resume.
"""
for tracer in self.tracers:
tracer.stop()
threading.settrace(None)
def stop(self):
if self.started:
sys.settrace(None)
if hasattr(threading, 'settrace'):
threading.settrace(None)
self.started = False
def start(self):
assert not self.started, "can't start if already started"
if not self.donothing:
sys.settrace = settrace
sys.settrace(self.globaltrace)
threading.settrace(self.globaltrace)
self.started = True
def thread_wrapper(self):
secs = getattr(test_func, '__timeout__', 0) or \
getattr(self.__class__, '__timeout__', 0) or \
_TimeoutThread.timeout
if secs <= 0:
return test_func(self)
try:
thread = _TimeoutThread(name=test_func.__qualname__,
target=test_func,
args=(self, ))
threading.settrace(thread.global_trace)
thread.start()
thread.join(secs)
alive = thread.isAlive()
thread.kill()
# re-join to ensure thread completes any blocking operations. This is
# really only required because long blocking calls may result
# in sequential tests using RedirectStdIO not setting back correctly
thread.join()
finally:
threading.settrace(None)
if thread.exc_info is not None:
raise thread.exc_info[1].with_traceback(thread.exc_info[2])
if alive:
raise unittest.SkipTest(
f'Function ran longer than {secs} second(s)')
return None
def start(self):
"""Start collecting trace information."""
if self._collectors:
self._collectors[-1].pause()
self._collectors.append(self)
#print("Started: %r" % self._collectors, file=sys.stderr)
# Check to see whether we had a fullcoverage tracer installed.
traces0 = []
if hasattr(sys, "gettrace"):
fn0 = sys.gettrace()
if fn0:
tracer0 = getattr(fn0, '__self__', None)
if tracer0:
traces0 = getattr(tracer0, 'traces', [])
# Install the tracer on this thread.
fn = self._start_tracer()
for args in traces0:
(frame, event, arg), lineno = args
try:
fn(frame, event, arg, lineno=lineno)
except TypeError:
raise Exception(
"fullcoverage must be run with the C trace function.")
# Install our installation tracer in threading, to jump start other
# threads.
threading.settrace(self._installation_trace)
def set(self, function):
""" Set a new function in sys.setprofile.
"""
if has_threading:
threading.settrace(function)
sys.settrace(function)
def main(argv=None):
''' Main entry point '''
args = parse_args(argv)
print(f'Running torch {torch.version.__version__}')
profile_cuda_memory = args.config.cuda.profile_cuda_memory
pin_memory = 'cuda' in args.device.type and not profile_cuda_memory
dataloader = get_dataloader(args.config.data,
args.seed_fn,
pin_memory,
args.num_devices,
shuffle=args.shuffle)
print(dataloader.dataset.stats)
model = args.model(args.config.model, dataloader.dataset)
action = args.action(args.action_config, model, dataloader, args.device)
if args.action_type == 'train' and args.action_config.early_stopping:
args.config.data.split = 'valid'
args.config.data.max_examples = 0
action.validation_dataloader = get_dataloader(args.config.data,
args.seed_fn,
pin_memory,
args.num_devices,
shuffle=args.shuffle)
if args.config.cuda.profile_cuda_memory:
print('Profiling CUDA memory')
memory_profiler = profile.CUDAMemoryProfiler(
action.modules.values(), filename=profile_cuda_memory)
sys.settrace(memory_profiler)
threading.settrace(memory_profiler)
step = 0
epoch = 0
if args.restore:
restore_modules = {
module_name: module
for module_name, module in action.modules.items()
if module_name not in args.reset_parameters
}
epoch, step = restore(args.restore,
restore_modules,
num_checkpoints=args.average_checkpoints,
map_location=args.device.type,
strict=not args.reset_parameters)
model.reset_named_parameters(args.reset_parameters)
if 'step' in args.reset_parameters:
step = 0
epoch = 0
args.experiment.set_step(step)
with ExitStack() as stack:
stack.enter_context(profiler.emit_nvtx(args.config.cuda.profile_cuda))
stack.enter_context(set_detect_anomaly(args.detect_anomalies))
action(epoch, args.experiment, args.verbose)
def set_trace(on=True): if on: t = trace_handler() threading.settrace(t.event_handler) sys.settrace(t.event_handler) else: sys.settrace(None) threading.settrace(None)
def trace(self, predicate):
self._handler = predicate
if self.threading_support:
self._threading_previous = getattr(threading, '_trace_hook', None)
threading.settrace(self)
self._previous = sys.gettrace()
sys.settrace(self)
return self
def stop_trace(self, threading_too=False):
sys.settrace(None)
frame = sys._getframe().f_back
while frame and frame is not self.botframe:
del frame.f_trace
frame = frame.f_back
if threading_too:
threading.settrace(None)
def set_trace(on=True): if on: t = trace_handler() threading.settrace(t.event_handler) sys.settrace(t.event_handler) else: sys.settrace(None) threading.settrace(None)
def stop_trace(self, threading_too=False):
sys.settrace(None)
frame = sys._getframe().f_back
while frame and frame is not self.botframe:
del frame.f_trace
frame = frame.f_back
if threading_too:
threading.settrace(None)
def wrapper(*args, **kwargs):
trace_func = sys.gettrace()
try:
sys.settrace(None)
threading.settrace(None)
return f(*args, **kwargs)
finally:
sys.settrace(trace_func)
threading.settrace(trace_func)
async def configure_tracing(self,
span_name,
span_kind=SERVER,
tags=None,
annotations=None
) -> Optional[ConsoleMeTracerObject]:
if not config.get("tracing.enabled", False):
return
if not random() * 100 <= config.get("tracing.sample_rate",
0.1): # nosec
return
if not tags:
tags = []
if not annotations:
annotations = []
zipkin_address = config.get(
"tracing.zipkin_address",
"http://127.0.0.1:9411/api/v2/spans").format(
region=config.region, environment=config.get("environment"))
endpoint = az.create_endpoint(
config.get("tracing.application_name", "consoleme"))
# The tracer's sample rate is 100% because we are pre-sampling our requests
self.tracer = await az.create(zipkin_address,
endpoint,
sample_rate=1.0)
self.primary_span = self.tracer.new_trace(sampled=True)
self.headers = self.primary_span.context.make_headers()
self.log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"message": "Starting trace",
"trace_id": self.primary_span.context.trace_id,
"zipkin_address": zipkin_address,
"tags": tags,
"hostname": config.hostname,
}
log.debug(self.log_data)
self.primary_span.kind(span_kind)
self.primary_span.start()
self.primary_span.name(span_name)
self.primary_span.tag("HOSTNAME", config.hostname)
for k, v in tags.items():
self.primary_span.tag(k, v)
for annotation in annotations:
self.primary_span.annotate(annotation)
# Configure sys/threading.settrace to use our trace_calls function for tracing
# Note: This is expensive, and should definitely not run for every request
sys.settrace(self.trace_calls)
threading.settrace(self.trace_calls)
return ConsoleMeTracerObject(primary_span=self.primary_span,
tracer=self.tracer,
headers=self.headers)
def opt_spew(self):
"""Print an insanely verbose log of everything that happens.
Useful when debugging freezes or locks in complex code."""
sys.settrace(util.spewer)
try:
import threading
except ImportError:
return
threading.settrace(util.spewer)
def run_trace(self):
"""
Context Manager for tracing
"""
sys.settrace(self) # noqa: DUO111
threading.settrace(self)
yield self
sys.settrace(None) # noqa: DUO111
threading.settrace(None)
def opt_spew(self):
"""Print an insanely verbose log of everything that happens.
Useful when debugging freezes or locks in complex code."""
sys.settrace(util.spewer)
try:
import threading
except ImportError:
return
threading.settrace(util.spewer)
def stop(self):
if self.started:
sys.settrace(None)
if hasattr(threading, 'settrace'):
threading.settrace(None)
self.started = False
self.stop_section()
def wrapper(*args, **kwargs):
trace_func = sys.gettrace()
try:
sys.settrace(None)
threading.settrace(None)
return f(*args, **kwargs)
finally:
sys.settrace(trace_func)
threading.settrace(trace_func)
def pause(self):
"""Pause tracing, but be prepared to `resume`."""
for tracer in self.tracers:
tracer.stop()
stats = tracer.get_stats()
if stats:
print("\nCoverage.py tracer stats:")
for k in sorted(stats.keys()):
print("%16s: %s" % (k, stats[k]))
threading.settrace(None)
def stop(self):
"""
Stop tracing. Reinstalls the :attr:`~hunter.tracer.Tracer.previous` tracer.
"""
if self._handler is not None:
sys.settrace(self._previous)
self._handler = self._previous = None
if self.threading_support is None or self.threading_support:
threading.settrace(self._threading_previous)
self._threading_previous = None
def pause(self):
"""Pause tracing, but be prepared to `resume`."""
for tracer in self.tracers:
tracer.stop()
stats = tracer.get_stats()
if stats:
print("\nCoverage.py tracer stats:")
for k in sorted(stats.keys()):
print("%16s: %s" % (k, stats[k]))
threading.settrace(None)
def start(self):
"""
Start recording.
"""
if not self.started:
self.started = True
sys.settrace(self.g)
if hasattr(threading, 'settrace'):
threading.settrace(self.g)
def start(self):
"""Start collecting trace information."""
if self._collectors:
self._collectors[-1].pause()
self._collectors.append(self)
# Install the tracer on this thread.
self._start_tracer()
# Install our installation tracer in threading, to jump start other
# threads.
threading.settrace(self._installation_trace)
def pause(self):
for tracer in self.tracers:
tracer.stop()
stats = tracer.get_stats()
if stats:
print '\nCoverage.py tracer stats:'
for k in sorted(stats.keys()):
print '%16s: %s' % (k, stats[k])
threading.settrace(None)
def start(self):
"""Start collecting trace information."""
if self._collectors:
self._collectors[-1].pause()
self._collectors.append(self)
# Install the tracer on this thread.
self._start_tracer()
# Install our installation tracer in threading, to jump start other
# threads.
threading.settrace(self._installation_trace)
def pause(self):
for tracer in self.tracers:
tracer.stop()
stats = tracer.get_stats()
if stats:
print '\nCoverage.py tracer stats:'
for k in sorted(stats.keys()):
print '%16s: %s' % (k, stats[k])
threading.settrace(None)
def stop_trace(self, frame):
if self.enable:
self.enable = False
tf = self.tracefunc_stack.pop()
while frame:
frame.f_trace = tf
frame = frame.f_back
sys.settrace(tf)
threading.settrace(tf)
def _symbolicExecFunction(self): threading.settrace(self.trace_func) sys.settrace(self.trace_func) try: self.inside_tracing_code = False result = self.function_to_be_traced(**self.arguments) finally: self.inside_tracing_code = True sys.settrace(None) threading.settrace(None) return result
def runctx(self, cmd, globals=None, locals=None):
if globals is None: globals = {}
if locals is None: locals = {}
if not self.donothing:
sys.settrace(self.globaltrace)
threading.settrace(self.globaltrace)
try:
exec cmd in globals, locals
finally:
if not self.donothing:
sys.settrace(None)
threading.settrace(None)
def set_dispatcher(self):
# print "######## set dispatcher ##########"
self.can_debug = False
frame = sys._getframe().f_back
self.reset()
while frame:
frame.f_trace = self.trace_dispatch
self.botframe = frame
frame = frame.f_back
self.set_continue()
sys.settrace(self.trace_dispatch)
threading.settrace(self.trace_dispatch)
def run(self, cmd):
import __main__
dict = __main__.__dict__
if not self.donothing:
threading.settrace(self.globaltrace)
sys.settrace(self.globaltrace)
try:
exec cmd in dict, dict
finally:
if not self.donothing:
sys.settrace(None)
threading.settrace(None)
def __init__(self, send_info, project_root):
self.project_root = project_root
if send_info.isdigit():
self.sender = _SocketSender(int(send_info))
else:
self.sender = _FileSender(send_info)
def global_trace(frame, event, arg):
# HACK: Ignoring out->in calls
# This might lose some information
if self._is_an_interesting_call(frame):
return self.on_function_call
sys.settrace(global_trace)
threading.settrace(global_trace)
def recover(self):
""" Unset the current function in the sys.settrace.
If available the previous method is recovered in settrace. A
RuntimeError is raised if the `previous` attribute does not exist.
"""
if hasattr(self, 'previous'):
sys.settrace(self.previous)
if has_threading:
threading.settrace(self.previous)
del self.previous
else:
raise RuntimeError('A profile function has not been set')
def stop(self):
"""Stop collecting trace information."""
assert self._collectors
assert self._collectors[-1] is self
for tracer in self.tracers:
tracer.stop()
self.tracers = []
threading.settrace(None)
# Remove this Collector from the stack, and resume the one underneath
# (if any).
self._collectors.pop()
if self._collectors:
self._collectors[-1].resume()
def profileOn(fnames):
global filenames, files, linecounts
filenames = fnames
files = {}
linecounts = {}
for f in fnames:
files[f] = mydict()
linecounts[f] = mydict()
#lStore.tPrev = time.time()
sys.settrace(te)
threading.settrace(te)
def replace(self, function):
""" Set a new function in sys.settrace.
If the function has been already set and it is not the same as before
then RuntimeError is raised.
"""
if hasattr(self, "previous"):
if function != sys.gettrace():
raise RuntimeError("Cannot replace profile function more than once")
return
else:
self.previous = sys.gettrace()
if has_threading:
threading.settrace(function)
sys.settrace(function)
def tracing(self, traceon):
"""Turn tracing on or off.
@param traceon: tracing flag
"""
if traceon == self.traceon:
return
self.traceon = traceon
if traceon:
self.tracefile = open(XEND_TRACE_FILE, 'w+', 1)
self.traceindent = 0
sys.settrace(self.trace)
try:
threading.settrace(self.trace) # Only in Python >= 2.3
except:
pass