def test_existing_profiler(self, checker: TypeChecker):
"""
Test that an existing profiler function is chained with the type checker and restored after
the block is exited.
"""
def foo(a: int):
pass
def profiler(frame, event, arg):
nonlocal profiler_run_count
if event in ('call', 'return'):
profiler_run_count += 1
if old_profiler:
old_profiler(frame, event, arg)
profiler_run_count = 0
old_profiler = sys.getprofile()
sys.setprofile(profiler)
try:
with checker, pytest.warns(TypeWarning) as record:
foo(1)
foo('x')
assert sys.getprofile() is profiler
finally:
sys.setprofile(old_profiler)
assert profiler_run_count
assert len(record) == 1
def test_tracer_install_on_start(tracer):
# initialize and start the tracer, and verify that it installs a
# profiler
tracer.start()
assert sys.getprofile() == tracer._trace
assert str(sys.getprofile()) == str(tracer._trace)
def test_setprofile():
profiler = TracingProfiler()
assert sys.getprofile() is None
with profiler:
assert sys.getprofile() == profiler._profile
assert sys.getprofile() is None
sys.setprofile(lambda *x: x)
with pytest.raises(RuntimeError):
profiler.start()
sys.setprofile(None)
def test_profile_enable_disable(self):
prof = self.profilerclass()
# Make sure we clean ourselves up if the test fails for some reason.
self.addCleanup(prof.disable)
prof.enable()
self.assertIs(sys.getprofile(), prof)
prof.disable()
self.assertIs(sys.getprofile(), None)
def test_profile_enable_disable(self):
prof = self.profilerclass()
# Make sure we clean ourselves up if the test fails for some reason.
self.addCleanup(prof.disable)
prof.enable()
self.assertIs(sys.getprofile(), prof)
prof.disable()
self.assertIs(sys.getprofile(), None)
def test_setprofile():
profiler = TracingProfiler()
assert sys.getprofile() is None
with profiler:
assert sys.getprofile() == profiler._profile
assert sys.getprofile() is None
sys.setprofile(lambda *x: x)
with pytest.raises(RuntimeError):
profiler.start()
sys.setprofile(None)
def test_error_when_set_multiple(self):
self.monitor._profile.replace(self.bar)
self.assertIs(sys.getprofile(), self.bar)
self.monitor._profile.replace(self.bar)
self.assertIs(sys.getprofile(), self.bar)
self.monitor._profile.recover()
self.monitor._profile.replace(self.bar)
self.assertIs(sys.getprofile(), self.bar)
with self.assertRaises(RuntimeError):
self.monitor._profile.replace(None)
self.assertIs(sys.getprofile(), self.bar)
self.monitor._profile.recover()
def test_error_when_set_multiple(self):
self.monitor._profile.replace(self.bar)
self.assertIs(sys.getprofile(), self.bar)
self.monitor._profile.replace(self.bar)
self.assertIs(sys.getprofile(), self.bar)
self.monitor._profile.recover()
self.monitor._profile.replace(self.bar)
self.assertIs(sys.getprofile(), self.bar)
with self.assertRaises(RuntimeError):
self.monitor._profile.replace(None)
self.assertIs(sys.getprofile(), self.bar)
self.monitor._profile.recover()
def test_profile_as_context_manager(self):
prof = self.profilerclass()
# Make sure we clean ourselves up if the test fails for some reason.
self.addCleanup(prof.disable)
with prof as __enter__return_value:
# profile.__enter__ should return itself.
self.assertIs(prof, __enter__return_value)
# profile should be set as the global profiler inside the
# with-block
self.assertIs(sys.getprofile(), prof)
# profile shouldn't be set once we leave the with-block.
self.assertIs(sys.getprofile(), None)
def test_profile_as_context_manager(self):
prof = self.profilerclass()
# Make sure we clean ourselves up if the test fails for some reason.
self.addCleanup(prof.disable)
with prof as __enter__return_value:
# profile.__enter__ should return itself.
self.assertIs(prof, __enter__return_value)
# profile should be set as the global profiler inside the
# with-block
self.assertIs(sys.getprofile(), prof)
# profile shouldn't be set once we leave the with-block.
self.assertIs(sys.getprofile(), None)
def run_model(clf, clf_name, model_params, df, prices, dataset_name,
magic_number, mode, trading_params, dates):
start = time()
if tracing:
pro_f = sys.getprofile()
sys.setprofile(None)
df_trade = train(df,
attrs,
clf,
clf_name,
model_params,
mode,
magic_number,
dates=dates,
dataset_name=dataset_name,
trading_params=trading_params)
indices = sorted([day for day in list(set(df_trade.index.values))])
portfolios = model_trade(df_trade,
indices=indices,
prices=prices,
clf_name=clf_name,
trading_params=trading_params,
dates=dates)
total_time = time() - start
if tracing:
sys.setprofile(pro_f)
return (portfolios, total_time)
def trace_calls_init_with_config(config: 'PostgresConfig') -> Iterator[None]:
"""Enable call tracing for a block of code"""
old_trace = sys.getprofile()
trace_logger = config.trace_logger()
class_trace_logger = config.class_trace_logger()
sys.setprofile(
CallTracer(logger=trace_logger,
class_logger=class_trace_logger,
code_filter=config.code_filter(),
sample_rate=config.sample_rate(),
skip_private_properties=config.skip_private_properties,
skip_private_methods=config.skip_private_methods))
if config.skip_private_properties:
print('Skipping private properties')
if config.skip_private_methods:
print('Skipping private methods')
try:
yield
finally:
sys.setprofile(old_trace)
trace_logger.flush()
if hasattr(trace_logger, 'remove_duplicates'):
trace_logger.remove_duplicates()
class_trace_logger.flush()
if hasattr(trace_logger, 'remove_duplicates'):
trace_logger.remove_duplicates()
def test_tracer_should_remove_on_stop(tracer):
# first start the tracer
tracer.start()
# then stop the tracer and assert that the profiler has been removed
tracer.stop()
assert sys.getprofile() is None
def test_same_object(self):
def foo(*args):
...
sys.setprofile(foo)
del foo
sys.setprofile(sys.getprofile())
def setUp(self):
super(TestTracingProperties, self).setUp()
if sys.gettrace() or sys.getprofile():
self.skipTest("Trace or profile function already active")
self.tracecount = 0
self.addCleanup(sys.settrace, None)
self.addCleanup(sys.setprofile, None)
def run_random(clf, clf_name, model_params, df, prices, dataset_name,
magic_number, mode, trading_params, dates):
start = time()
if tracing:
pro_f = sys.getprofile()
sys.setprofile(None)
start_date = np.datetime64(dates[0])
final_date = np.datetime64(dates[1])
indices = sorted([
day for day in list(set(df.index.values))
if start_date <= day <= final_date
])
# we add the offset of the training data which ain't required for Graham
indices = indices[magic_number:]
indices = [
indices[i]
for i in range(0, len(indices), trading_params['trade_frequency'])
]
print("Running debug with dataset: %s" % dataset_name)
portfolios = random_trade(df_trade=df,
indices=indices,
prices=prices,
clf_name=clf_name,
trading_params=trading_params,
dates=dates)
total_time = time() - start
if tracing:
sys.setprofile(pro_f)
return (portfolios, total_time)
def __init__(self, metascript):
# Indicates when activations should be collected
# (only after the first call to the script)
self.enabled = False
# User files
self.script = metascript.path
self.paths = metascript.paths
# Which function types ('main', 'package' or 'all')
# should be considered for the threshold
self.context = metascript.context
# How deep we want to go when capturing function activations?
self.depth_threshold = metascript.depth
# How deep we want to go beyond our context
self.non_user_depth_threshold = metascript.non_user_depth
# Object serializer function
self.serialize = metascript.serialize
self.metascript = metascript
self.trial_id = metascript.trial_id
self.event_map = defaultdict(lambda: self.trace_empty, {})
self.default_profile = sys.getprofile()
self.default_trace = sys.gettrace()
self.argument_captor = ArgumentCaptor(self)
def get_fundamentals(symbols_list_name, start_date, end_date, resample_period):
if tracing:
pro_f = sys.getprofile()
sys.setprofile(None)
print("Loading fundamentals for %s [%s - %s] %s" %
(symbols_list_name, start_date, end_date, resample_period))
df_fund = FundamentalsCollector(symbols_list_name=symbols_list_name,
start_date=start_date,
end_date=end_date).collect()
df_fund = (df_fund.drop_duplicates([
'date', 'symbol'
], keep='first').assign(
date=lambda r: pd.to_datetime(r.date, format=DATE_FORMAT)
).set_index('date').groupby('symbol').resample(
resample_period).ffill().replace('nm', np.NaN).sort_index().assign(
bookvaluepershare=lambda r: pd.to_numeric(r.bookvaluepershare)))
df_fund = pd.concat([
pd.to_numeric(df_fund[col], errors='ignore') for col in df_fund.columns
],
axis=1)
if tracing:
sys.setprofile(pro_f)
return df_fund
def __call__(self, func):
"""
Decorator
"""
from ..libpytimemory import component_bundle
global _is_running
if self._use:
self._original_profiler_function = sys.getprofile()
_is_running = True
component_bundle.reset()
component_bundle.configure(self.components, self._flat_profile,
self._timeline_profile)
@wraps(func)
def function_wrapper(*args, **kwargs):
if self._use:
sys.setprofile(_profiler_function)
_ret = func(*args, **kwargs)
if self._use:
sys.setprofile(self._original_profiler_function)
return _ret
_ret = function_wrapper
if self._use:
_is_running = False
return _ret
def __init__(self, components=[], flat=False, timeline=False, *args, **kwargs):
"""
Arguments:
- components [list of strings] : list of timemory components
- flat [bool] : enable flat profiling
- timeline [bool] : enable timeline profiling
"""
from ..libpytimemory import settings
global _records
global _include_line
global _include_filepath
global _full_filepath
global _counter
global _skip_counts
global _start_events
global _stop_events
global _is_running
_trace = settings.trace_components
_profl = settings.profiler_components
_components = _profl if _trace is None else _trace
self._original_profiler_function = sys.getprofile()
self._use = (not _is_running and Profiler.is_enabled() is True)
self._flat_profile = (settings.flat_profile or flat)
self._timeline_profile = (settings.timeline_profile or timeline)
self.components = components + _components.split(",")
if len(self.components) == 0:
self.components += ["wall_clock"]
os.environ["TIMEMORY_PROFILER_COMPONENTS"] = ",".join(self.components)
def get_info_on_next_invocation(self, func, callback, needed):
"""
func is a tuple (filename, funcname)
needed is a list/tuple of NeededInfo objects
"""
if isinstance(needed, NeededInfo):
needed = (needed,)
request = WatchRequest(needed, {}, callback)
if func in self.watching_funcs:
self.watching_funcs[func].requests.append(request)
else:
self.watching_funcs[func] = FuncWatchStatus(func, requests=[request])
fws = self.watching_funcs[func]
for ni in needed:
hsh = 0
for i in range(ni.stmt_idx+1):
hsh += hash(ni.stmt_sequence[i])
#cprint ("adding to hash %s: %i"%(str(ni.stmt_sequence[i]), hsh), 'green')
if hsh in fws.ni_hashmap:
fws.ni_hashmap[hsh].append((request,ni))
else:
fws.ni_hashmap[hsh] = [(request,ni)]
cprint("finishing up get_info_on_next_invocation", 'red')
# XXX Why do I need to use both settrace and setprofile? settrace doesn't
# XXX trigger on return events, and setprofile doesn't trigger on line events...
self.old_trace, self.old_profile = sys.gettrace(), sys.getprofile()
sys.settrace(self.trace_cb)
sys.setprofile(self.profile_cb)
def __init__(self, metascript):
# Indicates when activations should be collected
# (only after the first call to the script)
self.enabled = False
# User files
self.script = metascript.path
self.paths = metascript.paths
# Which function types ('main', 'package' or 'all')
# should be considered for the threshold
self.context = metascript.context
# How deep we want to go when capturing function activations?
self.depth_threshold = metascript.depth
# How deep we want to go beyond our context
self.non_user_depth_threshold = metascript.non_user_depth
# Object serializer function
self.serialize = metascript.serialize
self.metascript = metascript
self.trial_id = metascript.trial_id
self.event_map = defaultdict(lambda: self.trace_empty, {})
self.default_profile = sys.getprofile()
self.default_trace = sys.gettrace()
self.argument_captor = ArgumentCaptor(self)
def trace(self, predicate):
"""
Starts tracing with the given callable.
Args:
predicate (callable that accepts a single :obj:`~hunter.event.Event` argument):
Return:
self
"""
self._handler = predicate
if self.profiling_mode:
if self.threading_support is None or self.threading_support:
self._threading_previous = getattr(threading, '_profile_hook',
None)
threading.setprofile(self)
self._previous = sys.getprofile()
sys.setprofile(self)
else:
if self.threading_support is None or self.threading_support:
self._threading_previous = getattr(threading, '_trace_hook',
None)
threading.settrace(self)
self._previous = sys.gettrace()
sys.settrace(self)
return self
def wrapper(*args, **kwargs):
caller = inspect.getouterframes(
inspect.currentframe())[1] # pick up the caller
parsed = ast.parse(caller[4][0],
mode="single") # parse the calling line
arg_map = {
} # a map for our tracked args to establish global <=> local link
for node in ast.walk(parsed): # traverse the parsed code...
# and look for a call to our wrapped function
if isinstance(node, ast.Call) and node.func.id == funct.__name__:
# loop through all positional arguments of the wrapped function
for pos, var in enumerate(funct.func_code.co_varnames):
try: # and try to find them in the captured call
if isinstance(node.args[pos],
ast.Name): # named argument!
arg_map[var] = node.args[pos].id # add to our map
except IndexError:
break # no more passed arguments
break # no need for further walking through the ast tree
def trace(frame, evt, arg): # a function to capture the wrapped locals
if evt == "return": # we're only interested in our function return
for arg in arg_map: # time to update our caller frame
caller[0].f_locals[arg_map[arg]] = frame.f_locals.get(
arg, None)
profile = sys.getprofile() # in case something else is doing profiling
sys.setprofile(trace) # turn on profiling of the wrapped function
try:
return funct(*args, **kwargs)
finally:
sys.setprofile(profile) # reset our profiling
def __init__(self):
print('looking at some sys functions')
print(sys.api_version)
print(sys.argv)
print(sys.base_exec_prefix)
print(sys.base_prefix)
print(sys.builtin_module_names)
print(sys.byteorder)
print(sys.copyright)
print(sys.dllhandle)
print(sys.exc_info())
# print(sys.exc_traceback)
print(sys.executable)
print(sys.path)
print(sys.maxsize)
print(sys.platform)
print(sys.flags)
print(sys.float_info)
print(sys.float_repr_style)
print(sys._framework)
print(sys.getdefaultencoding())
print(sys.getwindowsversion())
print(sys.getallocatedblocks())
print(sys.getfilesystemencodeerrors())
# print(sys.getcheckinterval())
print(sys.getprofile())
print(sys.getrecursionlimit())
print(sys.getswitchinterval())
print(sys.gettrace())
print(sys._git)
print('\n')
print(sys.getsizeof(self))
def setUp(self):
if sys.gettrace() or sys.getprofile():
self.skipTest("Trace or profile function already active")
self.addCleanup(sys.settrace, None)
self.addCleanup(stackless.set_schedule_callback, None)
self.schedule_callback_exc = []
self.unexpected_trace_events = []
self.m = mutex()
def setUp(self):
if sys.gettrace() or sys.getprofile():
self.skipTest("Trace or profile function already active")
self.addCleanup(sys.settrace, None)
self.addCleanup(stackless.set_schedule_callback, None)
self.schedule_callback_exc = []
self.unexpected_trace_events = []
self.m = mutex()
def replace(self, function):
""" Set a new function in sys.setprofile.
If the function has been already set and it is not the same as before
then RuntimeError is raised.
"""
if hasattr(self, 'previous'):
if function != sys.getprofile():
raise RuntimeError('Cannot replace profile function more than '
'once')
return
else:
self.previous = sys.getprofile()
if has_threading:
threading.setprofile(function)
sys.setprofile(function)
def replace(self, function):
""" Set a new function in sys.setprofile.
If the function has been already set and it is not the same as before
then RuntimeError is raised.
"""
if hasattr(self, 'previous'):
if function != sys.getprofile():
raise RuntimeError(
'Cannot replace profile function more than once')
return
else:
self.previous = sys.getprofile()
if has_threading:
threading.setprofile(function)
sys.setprofile(function)
def start(self):
if sys.getprofile() is not None:
raise RuntimeError('Another profiler already registered.')
self._running = True
sys.setprofile(self._profile)
threading.setprofile(self._profile)
self.timer.start()
self.stats.record_starting(time.clock())
def task(self, expected_trace_function=None, expected_profile_function=None):
# this task tests that the trace/profile function has the
# expected value
tasklet = stackless.current
self.assertIs(sys.gettrace(), expected_trace_function)
self.assertIs(tasklet.trace_function, expected_trace_function)
self.assertIs(sys.getprofile(), expected_profile_function)
self.assertIs(tasklet.profile_function, expected_profile_function)
def start(self):
if sys.getprofile() is not None:
raise RuntimeError('Another profiler already registered.')
self._running = True
sys.setprofile(self._profile)
threading.setprofile(self._profile)
self.timer.start()
self.stats.record_starting(time.clock())
def trace_execution(fn, args, save_to=None):
import inspect
if save_to:
os.environ["INTEL_SEA_SAVE_TO"] = save_to
itt = ITT("python")
if itt.lib:
file_id = itt.get_string_id("__FILE__")
line_id = itt.get_string_id("__LINE__")
module_id = itt.get_string_id("__MODULE__")
trace_execution.frames = {}
trace_execution.recurrent = False
high_part = 2 ** 32
def profiler(frame, event, arg): # https://pymotw.com/2/sys/tracing.html
if trace_execution.recurrent:
return
trace_execution.recurrent = True
task_id = id(frame.f_code)
if "call" in event:
if task_id in trace_execution.frames:
trace_execution.frames[task_id] += 1
else:
trace_execution.frames[task_id] = 1
task_id += trace_execution.frames[task_id] * high_part
name = frame.f_code.co_name + ((" (%s)" % arg.__name__) if arg else "")
if "self" in frame.f_locals:
cls = frame.f_locals["self"].__class__.__name__
name = cls + "." + name
# print event, name, task_id, arg
mdl = inspect.getmodule(frame)
itt.lib.itt_task_begin_overlapped(itt.domain, task_id, 0, itt.get_string_id(name), 0)
itt.lib.itt_metadata_add_str(itt.domain, task_id, file_id, frame.f_code.co_filename)
itt.lib.itt_metadata_add(itt.domain, task_id, line_id, frame.f_code.co_firstlineno)
if mdl:
itt.lib.itt_metadata_add_str(itt.domain, task_id, module_id, mdl.__name__)
elif "return" in event:
# print event, frame.f_code.co_name, task_id + trace_execution.frames[task_id] * high_part
if task_id in trace_execution.frames:
itt.lib.itt_task_end_overlapped(
itt.domain, 0, task_id + trace_execution.frames[task_id] * high_part
)
if trace_execution.frames[task_id] > 1:
trace_execution.frames[task_id] -= 1
else:
del trace_execution.frames[task_id]
trace_execution.recurrent = False
print trace_execution.frames
old_profiler = sys.getprofile()
sys.setprofile(profiler)
old_threading_profiler = threading.setprofile(profiler)
fn(*args)
sys.setprofile(old_profiler)
threading.setprofile(old_threading_profiler)
else:
fn(*args)
def testSetTraceOnCurrent(self):
tf = self.nullTraceFunc
tasklet = stackless.current
self.assertIsNone(sys.gettrace())
self.assertIsNone(sys.getprofile())
self.assertIsNone(tasklet.trace_function)
self.assertIsNone(tasklet.profile_function)
tasklet.trace_function = tf
self.assertIs(sys.gettrace(), tf)
self.assertIsNone(sys.getprofile())
self.assertIs(tasklet.trace_function, tf)
self.assertIsNone(tasklet.profile_function)
tasklet.trace_function = None
self.assertIsNone(sys.gettrace())
self.assertIsNone(sys.getprofile())
self.assertIsNone(tasklet.trace_function)
self.assertIsNone(tasklet.profile_function)
self.assertGreater(self.tracecount, 0)
def main():
i = 1500
orig_profile = sys.getprofile()
sys.setprofile(profile)
try:
while i > 0:
i = sub(i, 1)
finally:
sys.setprofile(orig_profile)
def testSetTraceOnCurrent(self):
tf = self.nullTraceFunc
tasklet = stackless.current
self.assertIsNone(sys.gettrace())
self.assertIsNone(sys.getprofile())
self.assertIsNone(tasklet.trace_function)
self.assertIsNone(tasklet.profile_function)
tasklet.trace_function = tf
self.assertIs(sys.gettrace(), tf)
self.assertIsNone(sys.getprofile())
self.assertIs(tasklet.trace_function, tf)
self.assertIsNone(tasklet.profile_function)
tasklet.trace_function = None
self.assertIsNone(sys.gettrace())
self.assertIsNone(sys.getprofile())
self.assertIsNone(tasklet.trace_function)
self.assertIsNone(tasklet.profile_function)
self.assertGreater(self.tracecount, 0)
def main():
i = 1500
orig_profile = sys.getprofile()
sys.setprofile(profile)
try:
while i > 0:
i = sub(i, 1)
finally:
sys.setprofile(orig_profile)
def task2(self, expected_trace_function=None, expected_profile_function=None):
# task switches tasklets and then tests that the trace/profile function has the
# expected value
stackless.schedule()
tasklet = stackless.current
self.assertIs(sys.gettrace(), expected_trace_function)
self.assertIs(tasklet.trace_function, expected_trace_function)
self.assertIs(sys.getprofile(), expected_profile_function)
self.assertIs(tasklet.profile_function, expected_profile_function)
def trace_execution(fn, args, save_to=None):
import inspect
if save_to:
os.environ['INTEL_SEA_SAVE_TO'] = save_to
itt = ITT("python")
if itt.lib:
file_id = itt.get_string_id('__FILE__')
line_id = itt.get_string_id('__LINE__')
module_id = itt.get_string_id('__MODULE__')
trace_execution.frames = {}
trace_execution.recurrent = False
high_part = 2**32
def profiler(frame, event, arg): # https://pymotw.com/2/sys/tracing.html
if trace_execution.recurrent:
return
trace_execution.recurrent = True
task_id = id(frame.f_code)
if 'call' in event:
if task_id in trace_execution.frames:
trace_execution.frames[task_id] += 1
else:
trace_execution.frames[task_id] = 1
task_id += trace_execution.frames[task_id] * high_part
name = frame.f_code.co_name + ((' (%s)' % arg.__name__) if arg else '')
if 'self' in frame.f_locals:
cls = frame.f_locals['self'].__class__.__name__
name = cls + "." + name
# print event, name, task_id, arg
mdl = inspect.getmodule(frame)
itt.lib.itt_task_begin_overlapped(itt.domain, task_id, 0, itt.get_string_id(name), 0)
itt.lib.itt_metadata_add_str(itt.domain, task_id, file_id, frame.f_code.co_filename)
itt.lib.itt_metadata_add(itt.domain, task_id, line_id, frame.f_code.co_firstlineno)
if mdl:
itt.lib.itt_metadata_add_str(itt.domain, task_id, module_id, mdl.__name__)
elif 'return' in event:
# print event, frame.f_code.co_name, task_id + trace_execution.frames[task_id] * high_part
if task_id in trace_execution.frames:
itt.lib.itt_task_end_overlapped(itt.domain, 0, task_id + trace_execution.frames[task_id] * high_part)
if trace_execution.frames[task_id] > 1:
trace_execution.frames[task_id] -= 1
else:
del trace_execution.frames[task_id]
itt.counter('MEMORY_USAGE', get_memory_usage())
trace_execution.recurrent = False
old_profiler = sys.getprofile()
sys.setprofile(profiler)
old_threading_profiler = threading.setprofile(profiler)
if fn:
fn(*args)
sys.setprofile(old_profiler)
threading.setprofile(old_threading_profiler)
elif fn:
fn(*args)
def task(self,
expected_trace_function=None,
expected_profile_function=None):
# this task tests that the trace/profile function has the
# expected value
tasklet = stackless.current
self.assertIs(sys.gettrace(), expected_trace_function)
self.assertIs(tasklet.trace_function, expected_trace_function)
self.assertIs(sys.getprofile(), expected_profile_function)
self.assertIs(tasklet.profile_function, expected_profile_function)
def start():
"""
Turn on memory profiling.
"""
global _trace_memory
if sys.getprofile() is not None:
raise RuntimeError("another profile function is already active.")
if _trace_memory is None:
raise RuntimeError("trace.setup() was not called before trace.start().")
sys.setprofile(_trace_memory)
def start():
"""
Turn on memory profiling.
"""
global _trace_memory
if sys.getprofile() is not None:
raise RuntimeError("another profile function is already active.")
if _trace_memory is None:
raise RuntimeError("trace.setup() was not called before trace.start().")
sys.setprofile(_trace_memory)
def stop(self):
"""
Removes the module profiler globally and from future threads.
"""
if sys.getprofile() != self.profiler:
logger.warning(
"ModuleProfiler: The currently enabled profile function was not ours - unbinding anyways"
)
threading.setprofile(None)
sys.setprofile(None)
def uninstall(self):
"""
Deactivate this tracer.
If another profile hook was installed after this tracer was installed,
nothing will happen. If a different profile hook was installed prior to
calling ``install()``, it will be restored.
"""
if sys.getprofile() == self._trace:
sys.setprofile(self._wrapped_profiler)
def uninstall(self):
"""
Deactivate this tracer.
If another profile hook was installed after this tracer was installed,
nothing will happen. If a different profile hook was installed prior to
calling ``install()``, it will be restored.
"""
if sys.getprofile() == self._trace:
sys.setprofile(self._wrapped_profiler)
def setUp(self):
self.old = sys.getprofile()
def foo():
pass
def bar(frame, event, arg):
pass
self.foo = foo
self.bar = bar
self.monitor = MockNativeMonitor()
def add(self, gl):
if id(gl) not in self.gl:
# don't know about this module / globals yet
if not self.gl:
# install the profiler / cleanup handler, but keep the old one
self.op = sys.getprofile()
sys.setprofile(self)
# keep the reference for later
self.gl[id(gl)] = gl
def collect_events(func):
events = []
def tracer(frame, event, args):
events.append((frame.f_code, event))
old_tracer = sys.getprofile()
sys.setprofile(tracer)
try:
func()
finally:
sys.setprofile(old_tracer)
return events
def test_execute_and_capture_locals():
def func():
a=3
return a+4
def outer_func():
b=4
def nested():
return func()+b+5
return nested
out, local_vars = execute_and_capture_locals(func)
assert out == 3+4
assert local_vars == {'a': 3}
assert sys.getprofile() is None
out, local_vars = execute_and_capture_locals(outer_func())
assert out == 7+4+5
assert local_vars == {'b': 4, 'func': func}
assert sys.getprofile() is None
def profile_next_invocation(self, func, callback):
"""
func is a tuple (filename, funcname)
"""
fws = FuncWatchStatus(func, profiling=True)
self.watching_funcs[func] = fws
fws.profile_callback = callback
# XXX Why do I need to use both settrace and setprofile? settrace doesn't
# XXX trigger on return events, and setprofile doesn't trigger on line events...
self.old_trace, self.old_profile = sys.gettrace(), sys.getprofile()
sys.settrace(self.trace_cb)
sys.setprofile(self.profile_cb)
def __init__(self, metascript, depth_context, depth_threshold):
# Indicates when activations should be collected
# (only after the first call to the script)
self.enabled = False
self.script = metascript['path']
# which function types ('non-user' or 'all')
# should be considered for the threshold
self.depth_context = depth_context
# how deep we want to go when capturing function activations?
self.depth_threshold = depth_threshold
self.metascript = metascript
self.trial_id = metascript['trial_id']
self.event_map = defaultdict(lambda: self.trace_empty, {})
self.default_profile = sys.getprofile()
self.default_trace = sys.gettrace()
def run(self):
if sys.getprofile() is not None:
# NOTE: There's no threading.getprofile().
# The profiling function will be stored at threading._profile_hook
# but it's not documented.
raise RuntimeError('Another profiler already registered')
with deferral() as defer:
sys.setprofile(self._profile)
defer(sys.setprofile, None)
threading.setprofile(self._profile)
defer(threading.setprofile, None)
self.timer.start(self)
defer(self.timer.stop)
yield
self._times_entered.clear()
def install(self):
"""
Install this tracer as a global `profile hook
<https://docs.python.org/2/library/sys.html#sys.setprofile>`_.
The old profile hook, if one is set, will continue to be called by this
tracer.
"""
extant_profiler = sys.getprofile()
if isinstance(extant_profiler, cProfile.Profile):
raise RuntimeError(
"the pure-python Tracer is unable to compose over cProfile's "
"profile function; you must disable cProfile before "
"installing this Tracer.")
self._wrapped_profiler = extant_profiler
sys.setprofile(self._trace)
def _encodium_get_locals(func):
ret = None
def tracer(frame, event, arg):
nonlocal ret
if event == 'return':
ret = frame.f_locals.copy()
# tracer is activated on next call, return or exception
old_tracer = sys.getprofile()
sys.setprofile(tracer)
try:
# trace the function call
func()
finally:
# disable tracer and replace with old one
sys.setprofile(old_tracer)
return ret
def start():
"""
Turn on profiling.
"""
global _profile_start, _profile_setup, _call_stack, _inst_data
if _profile_start is not None:
print("profiling is already active.")
return
if not _profile_setup:
setup() # just do a default setup
_profile_start = etime()
_call_stack.append(('$total', _profile_start, None))
if '$total' not in _inst_data:
_inst_data['$total'] = [None, 0., 0]
if sys.getprofile() is not None:
raise RuntimeError("another profile function is already active.")
sys.setprofile(_instance_profile_callback)
def __call__(self, *args, **kwargs):
transaction = newrelic.api.transaction.current_transaction()
if not transaction:
return self._nr_next_object(*args, **kwargs)
#if transaction.coroutines:
# return self._nr_next_object(*args, **kwargs)
if not hasattr(sys, 'getprofile'):
return self._nr_next_object(*args, **kwargs)
profiler = sys.getprofile()
if profiler:
return self._nr_next_object(*args, **kwargs)
sys.setprofile(FunctionProfile(self._nr_depth))
try:
return self._nr_next_object(*args, **kwargs)
finally:
sys.setprofile(profiler)
def auditableFuncFactory(self, child_self, cls, *args, **kwargs):
func = self._func
if not hasattr(child_self, "audit"):
raise ValueError("auditble wrapper must be on auditable object")
class Tracer(object):
def __init__(self):
self.locals = {}
def __call__(self, frame, event, arg):
if event == "return":
self.locals = frame.f_locals.copy()
if child_self.audit:
# tracer is activated on next call, return or exception
tracer = Tracer()
old_profiler = sys.getprofile()
sys.setprofile(tracer)
try:
# trace the function call
res = func(child_self, *args, **kwargs)
finally:
# disable tracer and replace with old one
sys.setprofile(old_profiler)
# Filter out variables we don't want (marked with _)
v = tracer.locals
v.pop("self", None)
formula = None
for k in v.keys():
if k == "_formula":
formula = v[k]
if k[0] == "_":
v.pop(k, None)
return AuditResult(formula, res, v, func.func_name)
else:
return func(child_self, *args, **kwargs)
def trace_execution(fn, *args):
import inspect
itt = ITT("python")
if itt.lib:
trace_execution.depth = 0
file_id = itt.get_string_id('__FILE__')
line_id = itt.get_string_id('__LINE__')
module_id = itt.get_string_id('__MODULE__')
def profiler(frame, event, arg):
if 'call' in event:
trace_execution.depth += 1
name = frame.f_code.co_name
if 'self' in frame.f_locals:
cls = frame.f_locals['self'].__class__.__name__
name = cls + "." + name
mdl = inspect.getmodule(frame)
itt.lib.itt_task_begin(itt.domain, trace_execution.depth, 0, itt.get_string_id(name), 0)
itt.lib.itt_metadata_add_str(itt.domain, trace_execution.depth, file_id, frame.f_code.co_filename)
itt.lib.itt_metadata_add(itt.domain, trace_execution.depth, line_id, frame.f_code.co_firstlineno)
if mdl:
itt.lib.itt_metadata_add_str(itt.domain, trace_execution.depth, module_id, mdl.__name__)
if 'return' in event:
itt.lib.itt_task_end(itt.domain, 0)
trace_execution.depth -= 1
old_profiler = sys.getprofile()
sys.setprofile(profiler)
old_threading_profiler = threading.setprofile(profiler)
fn(*args)
sys.setprofile(old_profiler)
threading.setprofile(old_threading_profiler)
else:
fn(*args)
def test_setget(self):
def fn(*args):
pass
sys.setprofile(fn)
assert sys.getprofile() == fn