def test_print_formatting(self):
def a():
pass
def b():
a()
func_cols = {1: ("name", 48), 0: ("ncall", 5), 2: ("tsub", 8)}
thread_cols = {1: ("name", 48), 0: ("ttot", 8)}
yappi.start()
a()
b()
yappi.stop()
fs = yappi.get_func_stats()
cs = fs[1].children
ts = yappi.get_thread_stats()
# fs.print_all(out=sys.stderr, columns={1:("name", 70), })
# cs.print_all(out=sys.stderr, columns=func_cols)
# ts.print_all(out=sys.stderr, columns=thread_cols)
# cs.print_all(out=sys.stderr, columns={})
self.assertRaises(yappi.YappiError, fs.print_all, columns={1: ("namee", 9)})
self.assertRaises(yappi.YappiError, cs.print_all, columns={1: ("dd", 0)})
self.assertRaises(yappi.YappiError, ts.print_all, columns={1: ("tidd", 0)})
def test_filter(self):
def a(): pass
def b(): a()
def c(): b()
_TCOUNT = 5
ts = []
yappi.start()
for i in range(_TCOUNT):
t = threading.Thread(target=c)
t.start()
ts.append(t)
for t in ts:
t.join()
yappi.stop()
fstats = yappi.get_func_stats(filter={"ctx_id":9})
self.assertTrue(fstats.empty())
fstats = yappi.get_func_stats(filter={"ctx_id":0, "name":"c"}) # main thread
self.assertTrue(fstats.empty())
for i in range(1, _TCOUNT):
fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"a",
"ncall":1})
self.assertEqual(fstats.pop().ncall, 1)
fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"b"})
self.assertEqual(fstats.pop().ncall, 1)
fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"c"})
self.assertEqual(fstats.pop().ncall, 1)
def stop_profiler(self):
"""
Stop yappi and write the stats to the output directory.
Return the path of the yappi statistics file.
"""
if not self.profiler_running:
raise RuntimeError("Profiler is not running")
if not HAS_YAPPI:
raise RuntimeError("Yappi cannot be found. Plase install the yappi library using your preferred package "
"manager and restart Tribler afterwards.")
yappi.stop()
yappi_stats = yappi.get_func_stats()
yappi_stats.sort("tsub")
log_dir = os.path.join(self.session.config.get_state_dir(), 'logs')
file_path = os.path.join(log_dir, 'yappi_%s.stats' % self.profiler_start_time)
# Make the log directory if it does not exist
if not os.path.exists(log_dir):
os.makedirs(log_dir)
yappi_stats.save(file_path, type='callgrind')
yappi.clear_stats()
self.profiler_running = False
return file_path
def test_pause_resume(self):
yappi.set_context_id_callback(lambda: self.context_id)
yappi.set_clock_type('wall')
# Start in context 0.
self.context_id = 0
yappi.start()
time.sleep(0.08)
# Switch to context 1.
self.context_id = 1
time.sleep(0.05)
# Switch back to context 0.
self.context_id = 0
time.sleep(0.07)
yappi.stop()
t_stats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(2, len(t_stats))
self.assertEqual(0, t_stats[0].id)
self.assertEqual(2, t_stats[0].sched_count)
self.assertTrue(0.15 < t_stats[0].ttot < 0.3)
self.assertEqual(1, t_stats[1].id)
self.assertEqual(1, t_stats[1].sched_count)
# Context 1 was first scheduled 0.08 sec after context 0.
self.assertTrue(0.1 < t_stats[1].ttot < 0.2 )
def setUp(self):
# reset everything back to default
yappi.stop()
yappi.clear_stats()
yappi.set_clock_type('cpu') # reset to default clock type
yappi.set_context_id_callback(None)
yappi.set_context_name_callback(None)
def test_profile_single_context(self):
def id_callback():
return self.callback_count
def a():
pass
self.callback_count = 1
yappi.set_context_id_callback(id_callback)
yappi.start(profile_threads=False)
a() # context-id:1
self.callback_count = 2
a() # context-id:2
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, "a")
self.assertEqual(fsa.ncall, 1)
yappi.stop()
yappi.clear_stats()
self.callback_count = 1
yappi.start() # profile_threads=True
a() # context-id:1
self.callback_count = 2
a() # context-id:2
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, "a")
self.assertEqual(fsa.ncall, 2)
def test_callback(self):
self.context_id = 0
yappi.set_context_id_callback(lambda: self.context_id)
yappi.start()
a()
self.context_id = 1
a()
self.context_id = 2
a()
# Re-schedule context 1.
self.context_id = 1
a()
yappi.stop()
threadstats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(3, len(threadstats))
self.assertEqual(0, threadstats[0].id)
self.assertEqual(1, threadstats[1].id)
self.assertEqual(2, threadstats[2].id)
self.assertEqual(1, threadstats[0].sched_count)
self.assertEqual(2, threadstats[1].sched_count) # Context 1 ran twice.
self.assertEqual(1, threadstats[2].sched_count)
funcstats = yappi.get_func_stats()
self.assertEqual(4, utils.find_stat_by_name(funcstats, 'a').ncall)
def test_producer_consumer_with_queues(self):
# we currently just stress yappi, no functionality test is done here.
yappi.start()
import time
if utils.is_py3x():
from queue import Queue
else:
from Queue import Queue
from threading import Thread
WORKER_THREAD_COUNT = 50
WORK_ITEM_COUNT = 2000
def worker():
while True:
item = q.get()
# do the work with item
q.task_done()
q = Queue()
for i in range(WORKER_THREAD_COUNT):
t = Thread(target=worker)
t.daemon = True
t.start()
for item in range(WORK_ITEM_COUNT):
q.put(item)
q.join()# block until all tasks are done
#yappi.get_func_stats().sort("callcount").print_all()
yappi.stop()
def stop():
pages.require("/admin/settings.edit", noautoreturn=True)
pages.postOnly()
import yappi
if yappi.is_running():
yappi.stop()
raise cherrypy.HTTPRedirect("/settings/profiler")
def test_yappi_overhead(self):
import time
LOOP_COUNT = 10000
def a(): pass
def b():
for i in range(LOOP_COUNT): a()
t0 = time.time()
yappi.start()
b()
yappi.stop()
time_with_yappi = time.time() - t0
t0 = time.time()
b()
time_without_yappi = time.time() - t0
if time_without_yappi == 0:
time_without_yappi = 0.000001
# in latest v0.82, I calculated this as close to "7.0" in my machine.
# however, %83 of this overhead is coming from tickcount(). The other %17
# seems to have been evenly distributed to the internal bookkeeping
# structures/algorithms which seems acceptable. Note that our test only
# tests one function being profiled at-a-time in a short interval.
# profiling high number of functions in a small time
# is a different beast, (which is pretty unlikely in most applications)
# So as a conclusion: I cannot see any optimization window for Yappi that
# is worth implementing as we will only optimize %17 of the time.
sys.stderr.write("\r\nYappi puts %0.1f times overhead to the profiled application in average.\r\n" % \
(time_with_yappi / time_without_yappi))
def test_merge_load_different_clock_types(self):
import threading
yappi.start(builtins=True)
def a(): b()
def b(): c()
def c(): pass
t = threading.Thread(target=a)
t.start()
t.join()
yappi.get_func_stats().sort("name", "asc").save("ystats1.ys")
yappi.stop()
yappi.clear_stats()
yappi.start(builtins=False)
t = threading.Thread(target=a)
t.start()
t.join()
yappi.get_func_stats().save("ystats2.ys")
yappi.stop()
self.assertRaises(_yappi.error, yappi.set_clock_type, "wall")
yappi.clear_stats()
yappi.set_clock_type("wall")
yappi.start()
t = threading.Thread(target=a)
t.start()
t.join()
yappi.get_func_stats().save("ystats3.ys")
self.assertRaises(yappi.YappiError, yappi.YFuncStats().add("ystats1.ys").add, "ystats3.ys")
stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys"]).sort("name")
fsa = utils.find_stat_by_name(stats, "a")
fsb = utils.find_stat_by_name(stats, "b")
fsc = utils.find_stat_by_name(stats, "c")
self.assertEqual(fsa.ncall, 2)
self.assertEqual(fsa.ncall, fsb.ncall, fsc.ncall)
def test_concurrent_futures(self):
yappi.start()
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=5) as executor:
f = executor.submit(pow, 5, 2)
self.assertEqual(f.result(), 25)
time.sleep(1.0)
yappi.stop()
def _stop_profiling(filename, format):
logging.debug("Stopping CPU profiling")
with _lock:
if yappi.is_running():
yappi.stop()
stats = yappi.get_func_stats()
stats.save(filename, format)
yappi.clear_stats()
def wrapped(*args, **kwargs):
yappi.start()
result = func(*args, **kwargs)
yappi.stop()
prof_file = "%s.%s" % (func.__name__, time.time())
#prof_file = "callgrind.a.1"
yappi.get_func_stats().save(prof_file, "ystat")
return result
def stop(self):
if not yappi.is_running():
raise UsageError("CPU profiler is not running")
logging.info("Stopping CPU profiling")
yappi.stop()
stats = yappi.get_func_stats()
stats.save(self.filename, self.format)
yappi.clear_stats()
def cmd_stop(self, args):
"""stop profiling"""
print "Profile results:"
import yappi # We do the import here so that we won't barf if run normally and yappi not available
yappi.get_func_stats().print_all(
columns={0: ("name", 50), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)}
)
yappi.get_thread_stats().print_all()
yappi.stop()
def do_action(self):
action = self.cleaned_data['action']
if action == 'start':
yappi.start()
elif action == 'start_with_builtins':
yappi.start(builtins=True)
elif action == 'stop':
yappi.stop()
elif action == 'reset':
yappi.clear_stats()
return action
def profile():
"""
Code profiler using YAPPI
http://code.google.com/p/yappi
"""
import yappi
yappi.start()
start(False)
yappi.stop()
for pstat in yappi.get_stats(yappi.SORTTYPE_TSUB):
print pstat
def _stop_profiling(self):
with lock:
if not yappi.is_running():
raise http.Error(http.BAD_REQUEST, "profile is not running")
log.info("Stopping profiling, writing profile to %r",
self.config.profile.filename)
yappi.stop()
stats = yappi.get_func_stats()
stats.save(self.config.profile.filename, type="pstat")
yappi.clear_stats()
def main():
print('Main TID: {}'.format(gettid()))
args = Bunch(channel=None,
devices=[],
generate=False,
ui='main.ui')
yappi.start()
exit_value = epyqlib.__main__.main(args=args)
yappi.stop()
yappi.get_func_stats().save('yappi.stats', type='pstat')
yappi.get_thread_stats().print_all()
return exit_value
def test_subsequent_profile(self):
import threading
WORKER_COUNT = 5
def a(): pass
def b(): pass
def c(): pass
_timings = {"a_1":3,"b_1":2,"c_1":1,}
yappi.start()
def g(): pass
g()
yappi.stop()
yappi.clear_stats()
_yappi._set_test_timings(_timings)
yappi.start()
_dummy = []
for i in range(WORKER_COUNT):
t = threading.Thread(target=a)
t.start()
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=b)
t.start()
_dummy.append(t)
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=a)
t.start()
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=c)
t.start()
t.join()
yappi.stop()
yappi.start()
def f():
pass
f()
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, 'a')
fsb = utils.find_stat_by_name(stats, 'b')
fsc = utils.find_stat_by_name(stats, 'c')
self.assertEqual(fsa.ncall, 10)
self.assertEqual(fsb.ncall, 5)
self.assertEqual(fsc.ncall, 5)
self.assertEqual(fsa.ttot, fsa.tsub, 30)
self.assertEqual(fsb.ttot, fsb.tsub, 10)
self.assertEqual(fsc.ttot, fsc.tsub, 5)
# MACOSx optimizes by only creating one worker thread
self.assertTrue(len(yappi.get_thread_stats()) >= 2)
def profile_cpu_bound_program():
real_dog = DogStatsApi()
real_dog.reporter = NullReporter()
fake_dog = NullDogStatsApi()
for type_, dog in [('real', real_dog), ('fake', fake_dog)]:
print('\n\n\nTESTING %s\n\n' % type_)
dog.start()
program = CPUBoundProgram(dog)
yappi.start()
program.run()
yappi.print_stats(sort_type=yappi.SORTTYPE_TSUB, sort_order=yappi.SORTORDER_DESC)
yappi.stop()
yappi.clear_stats()
def test_no_stats_different_clock_type_load(self):
def a(): pass
yappi.start()
a()
yappi.stop()
yappi.get_func_stats().save("ystats1.ys")
yappi.clear_stats()
yappi.set_clock_type("WALL")
yappi.start()
yappi.stop()
stats = yappi.get_func_stats().add("ystats1.ys")
fsa = utils.find_stat_by_name(stats, 'a')
self.assertTrue(fsa is not None)
def finish(self):
from gi.repository import Gtk
if self.debug:
import yappi
yappi.stop()
yappi.convert2pstats(yappi.get_func_stats()).dump_stats(
'mfp-gui-funcstats.pstats')
log.log_func = None
if self.appwin:
self.appwin.quit()
self.appwin = None
Gtk.main_quit()
def stop(self):
if self._isStart:
yappi.stop()
yappi.set_context_id_callback(None)
self._isStart = False
if not os.path.exists(self._path):
os.mkdir(self._path)
stats = yappi.get_func_stats()
fullPath = self._path + "/callgrind." + self._serviceName
stats.save(fullPath, type='callgrind')
return 'success'
return 'fail'
def test_module_stress(self):
self.assertEqual(yappi.is_running(), False)
yappi.start()
yappi.clear_stats()
self.assertRaises(_yappi.error, yappi.set_clock_type, "wall")
yappi.stop()
yappi.clear_stats()
yappi.set_clock_type("cpu")
self.assertRaises(yappi.YappiError, yappi.set_clock_type, "dummy")
self.assertEqual(yappi.is_running(), False)
yappi.clear_stats()
yappi.clear_stats()
def main():
parser = arg_parser()
args = parser.parse_args()
if args.profile:
import yappi
yappi.start()
try:
run_analysis(parser, args)
finally:
yappi.stop()
stats = yappi.get_func_stats().sort('tsub').strip_dirs()
stats.print_all(out=sys.stderr, columns={0: ('name', 45), 1: ('ncall', 10), 2: ('tsub', 8), 3: ('ttot', 8), 4: ('tavg', 8)})
else:
run_analysis(parser, args)
def test_callback_non_integer(self):
self.callback_count = 0
def callback():
self.callback_count += 1
return None # Supposed to return an integer.
yappi.set_context_id_callback(callback)
yappi.start()
a()
a()
yappi.stop()
# Callback was cleared after first error.
self.assertEqual(1, self.callback_count)
def test_callback_error(self):
self.callback_count = 0
def callback():
self.callback_count += 1
raise Exception('callback error')
yappi.set_context_id_callback(callback)
yappi.start()
a()
a()
yappi.stop()
# Callback was cleared after first error.
self.assertEqual(1, self.callback_count)
def test_callback_non_string(self):
self.callback_count = 0
def callback():
self.callback_count += 1
return 1 # Supposed to return a string.
yappi.set_context_name_callback(callback)
yappi.start()
a()
a()
yappi.stop()
# Callback was cleared after first error.
self.assertEqual(1, self.callback_count)
def test_merge_load_different_clock_types(self):
yappi.start(builtins=True)
def a():
b()
def b():
c()
def c():
pass
t = threading.Thread(target=a)
t.start()
t.join()
yappi.get_func_stats().sort("name", "asc").save("ystats1.ys")
yappi.stop()
yappi.clear_stats()
yappi.start(builtins=False)
t = threading.Thread(target=a)
t.start()
t.join()
yappi.get_func_stats().save("ystats2.ys")
yappi.stop()
self.assertRaises(_yappi.error, yappi.set_clock_type, "wall")
yappi.clear_stats()
yappi.set_clock_type("wall")
yappi.start()
t = threading.Thread(target=a)
t.start()
t.join()
yappi.get_func_stats().save("ystats3.ys")
self.assertRaises(yappi.YappiError,
yappi.YFuncStats().add("ystats1.ys").add,
"ystats3.ys")
stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys"]).sort("name")
fsa = utils.find_stat_by_name(stats, "a")
fsb = utils.find_stat_by_name(stats, "b")
fsc = utils.find_stat_by_name(stats, "c")
self.assertEqual(fsa.ncall, 2)
self.assertEqual(fsa.ncall, fsb.ncall, fsc.ncall)
def test_basic_old_style(self):
@asyncio.coroutine
def a():
yield from async_sleep(0.1)
burn_io(0.1)
yield from async_sleep(0.1)
burn_io(0.1)
yield from async_sleep(0.1)
burn_cpu(0.3)
yappi.set_clock_type("wall")
yappi.start(builtins=True)
asyncio.get_event_loop().run_until_complete(a())
asyncio.get_event_loop().run_until_complete(a())
yappi.stop()
r1 = '''
..p/yappi/tests/test_asyncio.py:43 a 2 0.000118 1.604049 0.802024
async_sleep 6 0.000000 0.603239 0.100540
../yappi/tests/utils.py:126 burn_cpu 2 0.576313 0.600026 0.300013
..p/yappi/tests/utils.py:135 burn_io 4 0.000025 0.400666 0.100166
time.sleep 4 0.400641 0.400641 0.100160
'''
stats = yappi.get_func_stats()
self.assert_traces_almost_equal(r1, stats)
yappi.clear_stats()
yappi.set_clock_type("cpu")
yappi.start(builtins=True)
asyncio.get_event_loop().run_until_complete(a())
asyncio.get_event_loop().run_until_complete(a())
yappi.stop()
stats = yappi.get_func_stats()
r1 = '''
..p/yappi/tests/test_asyncio.py:43 a 2 0.000117 0.601170 0.300585
../yappi/tests/utils.py:126 burn_cpu 2 0.000000 0.600047 0.300024
async_sleep 6 0.000159 0.000801 0.000134
time.sleep 4 0.000169 0.000169 0.000042
'''
self.assert_traces_almost_equal(r1, stats)
def _capture_profile(fname=''):
if not fname:
yappi.set_clock_type('cpu')
# We need to set context to greenlet to profile greenlets
# https://bitbucket.org/sumerc/yappi/pull-requests/3
yappi.set_context_id_callback(lambda: id(greenlet.getcurrent()))
yappi.set_context_name_callback(
lambda: greenlet.getcurrent().__class__.__name__)
yappi.start()
else:
yappi.stop()
stats = yappi.get_func_stats()
# User should provide filename. This file with a suffix .prof
# will be created in temp directory.
try:
stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof')
stats.save(stats_file, "pstat")
except Exception as e:
print("Error while saving the trace stats ", str(e))
finally:
yappi.clear_stats()
def main():
stored_lengths = {}
max_length = (-1, -1)
max_range = 1000000
yappi.start()
for i in range(max_range):
print(f'i: {i}')
collatz = calculate_collatz([i])
length = sequence_length_dictionary(collatz, stored_lengths)
if length > max_length[0]:
max_length = (length, i)
for i in range(max_range):
print(f'i: {i}')
length = collatz_sequence_length(i)
if length > max_length[0]:
max_length = (length, i)
yappi.stop()
print(
f'The longest collatz sequence is {max_length[0]} with the starting value {max_length[1]}'
)
def _handle_sigusr2(sig, stack):
'''
Signal handler for SIGUSR2, only available on Unix-like systems
'''
try:
import yappi
except ImportError:
return
if yappi.is_running():
yappi.stop()
filename = 'callgrind.salt-{0}-{1}'.format(int(time.time()),
os.getpid())
destfile = os.path.join(tempfile.gettempdir(), filename)
yappi.get_func_stats().save(destfile, type='CALLGRIND')
if sys.stderr.isatty():
sys.stderr.write('Saved profiling data to: {0}\n'.format(destfile))
yappi.clear_stats()
else:
if sys.stderr.isatty():
sys.stderr.write('Profiling started\n')
yappi.start()
def test_temporary_lock_waiting(self):
import threading
import time
yappi.start()
_lock = threading.Lock()
def worker():
_lock.acquire()
try:
time.sleep(1.0)
finally:
_lock.release()
t1 = threading.Thread(target=worker)
t2 = threading.Thread(target=worker)
t1.start()
t2.start()
t1.join()
t2.join()
#yappi.get_func_stats().sort("callcount").print_all()
yappi.stop()
def test_recursive_function(self):
def a(n):
if (n <= 0):
return
burn_io_gevent(0.001)
burn_cpu(0.1)
a(n - 1)
a(n - 2)
def driver():
gls = []
for i in (3, 4):
gls.append(self.spawn_greenlet("recursive_%d" % (i), a, i))
for gl in gls:
gl.get()
yappi.set_clock_type("cpu")
yappi.start()
driver()
yappi.stop()
r1 = '''
tests/test_gevent.py:209 a 24/2 0.000407 1.102129 0.045922
../yappi/tests/utils.py:142 burn_cpu 11 0.000000 1.100660 0.100060
../tests/utils.py:154 burn_io_gevent 11 0.000159 0.001062 0.000097
..e-packages/gevent/hub.py:126 sleep 11 0.000903 0.000903 0.000082
tests/test_gevent.py:219 driver 1 0.000208 0.000467 0.000467
'''
stats = yappi.get_func_stats()
self.assert_traces_almost_equal(r1, stats)
gstats = yappi.get_greenlet_stats()
r2 = '''
Main/recursive_4 4 0.701283 5
Main/recursive_3 3 0.400664 5
Main 1 0.000439 3
'''
self.assert_ctx_stats_almost_equal(r2, gstats)
def test_recursive_coroutine(self):
@asyncio.coroutine
def a(n):
if n <= 0:
return
yield from async_sleep(0.1)
burn_cpu(0.1)
yield from a(n - 1)
yield from a(n - 2)
yappi.set_clock_type("cpu")
yappi.start()
asyncio.get_event_loop().run_until_complete(a(3))
yappi.stop()
r1 = '''
..p/yappi/tests/test_asyncio.py:11 a 9/1 0.000124 0.400667 0.044519
../yappi/tests/utils.py:126 burn_cpu 4 0.000000 0.400099 0.100025
async_sleep 4 0.000000 0.000444 0.000111
'''
stats = yappi.get_func_stats()
self.assert_traces_almost_equal(r1, stats)
def test_start_flags(self):
self.assertEqual(_yappi._get_start_flags(), None)
yappi.start()
def a():
pass
a()
self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0)
self.assertEqual(_yappi._get_start_flags()["profile_multicontext"], 1)
self.assertEqual(len(yappi.get_greenlet_stats()), 1)
yappi.stop()
yappi.clear_stats()
yappi.start(builtins=True,
profile_greenlets=True,
profile_threads=False)
self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 1)
self.assertEqual(_yappi._get_start_flags()["profile_multicontext"], 1)
self.assertEqual(len(yappi.get_greenlet_stats()), 1)
yappi.stop()
def single_analysis_with_yappi(ini, end=None):
"""Perform profiling analysis for each test case, separately.
Args:
ini (int): First test case number analyzed.
end (int): Last test case number analyzed.
config (Config): Configuration object for PyCallGraph.
"""
import yappi
end = end or ini
for num in range(ini, end + 1):
print("Running test case number", num, "in a single analysis.")
yappi.set_clock_type('cpu')
yappi.start(builtins=True)
_run_test_case(num)
yappi.stop()
stats = yappi.get_func_stats()
stats.save('yappi.callgrind', type='callgrind')
def test_recursive_greenlet(self):
def a(n):
if n <= 0:
return
burn_io_gevent(0.1)
burn_cpu(0.1)
g1 = self.spawn_greenlet("a_%d" % (n - 1), a, n - 1)
g1.get()
g2 = self.spawn_greenlet("a_%d" % (n - 2), a, n - 2)
g2.get()
yappi.start()
g = self.spawn_greenlet("a", a, 3)
g.get() # run until complete, report exception (if any)
yappi.stop()
r1 = '''
..p/yappi/tests/test_asyncio.py:11 a 9 0.000124 0.400667 0.044519
../yappi/tests/utils.py:126 burn_cpu 4 0.000000 0.400099 0.100025
sleep 4 0.000000 0.000444 0.000111
'''
stats = yappi.get_func_stats()
self.assert_traces_almost_equal(r1, stats)
gstats = yappi.get_greenlet_stats()
r2 = '''
Main/a/a_1 9 0.100588 3
Main/a/a_2 4 0.100588 3
Main/a 3 0.100584 3
Main/a/a_2/a_1 5 0.100549 3
Main 1 0.000356 2
Main/a/a_1/a_0 10 0.000046 1
Main/a/a_2/a_1/a_0 6 0.000044 1
Main/a/a_2/a_1/a_-1 7 0.000036 1
Main/a/a_2/a_0 8 0.000035 1
Main/a/a_1/a_-1 11 0.000029 1
'''
self.assert_ctx_stats_almost_equal(r2, gstats)
def _stop(self, config, current_time):
yappi.stop()
global_log.log(scalyr_logging.DEBUG_LEVEL_0, "Stopping CPU profiling")
stats = yappi.get_func_stats()
if os.path.exists(self._data_file_path):
os.remove(self._data_file_path)
# pylint bug https://github.com/PyCQA/pylint/labels/topic-inference
stats.save(self._data_file_path, "callgrind") # pylint: disable=no-member
lines = 0
# count the lines
f = open(self._data_file_path)
try:
for line in f:
lines += 1
finally:
f.close()
# write a status message to make it easy to find the end of each profile session
f = open(self._data_file_path, "a")
try:
f.write(
"\n# %s, %s clock, total lines: %d\n"
% (self._data_file_path, self._profile_clock, lines)
)
finally:
f.close()
yappi.clear_stats()
del stats
global_log.log(
scalyr_logging.DEBUG_LEVEL_0,
"CPU profiling data written to %s",
self._data_file_path,
)
def test_print_formatting(self):
def a():
pass
def b():
a()
func_cols = {
1: ("name", 48),
0: ("ncall", 5),
2: ("tsub", 8),
}
thread_cols = {
1: ("name", 48),
0: ("ttot", 8),
}
yappi.start()
a()
b()
yappi.stop()
fs = yappi.get_func_stats()
cs = fs[1].children
ts = yappi.get_thread_stats()
#fs.print_all(out=sys.stderr, columns={1:("name", 70), })
#cs.print_all(out=sys.stderr, columns=func_cols)
#ts.print_all(out=sys.stderr, columns=thread_cols)
#cs.print_all(out=sys.stderr, columns={})
self.assertRaises(yappi.YappiError,
fs.print_all,
columns={1: ("namee", 9)})
self.assertRaises(yappi.YappiError,
cs.print_all,
columns={1: ("dd", 0)})
self.assertRaises(yappi.YappiError,
ts.print_all,
columns={1: ("tidd", 0)})
def test_context_cbks_reset_to_default(self):
yappi.set_context_backend("greenlet")
yappi.set_context_backend("native_thread")
class ThreadA(threading.Thread):
def run(self):
burn_cpu(0.05)
def a():
pass
yappi.start()
t = ThreadA()
t.start()
t.join()
# Spawn a greenlet to test that greenlet context is not recognised
g = gevent.Greenlet(a)
g.start()
g.get()
yappi.stop()
tstats = yappi.get_thread_stats()
self.assertEqual(len(tstats), 2,
"Incorrect number of contexts captured")
# First stat should be of threadA since it is sorted by ttot
statsA = tstats[0]
self.assertEqual(statsA.tid, t.ident)
self.assertEqual(statsA.name, t.__class__.__name__)
statsMain = tstats[1]
main_thread = threading.current_thread()
self.assertEqual(statsMain.tid, main_thread.ident)
self.assertEqual(statsMain.name, main_thread.__class__.__name__)
def test_builtin_profiling(self):
import threading
def a():
import time
time.sleep(0.4) # is a builtin function
yappi.set_clock_type('wall')
yappi.start(builtins=True)
a()
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, 'sleep')
self.assertTrue(fsa is not None)
self.assertTrue(fsa.ttot > 0.3)
yappi.stop()
yappi.clear_stats()
def a():
pass
yappi.start()
t = threading.Thread(target=a)
t.start()
t.join()
stats = yappi.get_func_stats()
def stop(self):
"""
Stop Yappi and write the stats to the output directory.
Return the path of the statistics file.
"""
if not self._is_running:
raise RuntimeError("Profiler is not running")
yappi.stop()
yappi_stats = yappi.get_func_stats()
yappi_stats.sort('tsub', sort_order="desc")
log_dir = self.logs_dir
file_path = log_dir / f"yappi_{self._start_time}.stats"
# Make the log directory if it does not exist
if not log_dir.exists():
os.makedirs(log_dir)
yappi_stats.save(file_path, type='callgrind')
yappi.clear_stats()
self._is_running = False
return file_path
def test_subsequent_profile(self):
_timings = {"a_1": 1, "b_1": 1}
_yappi._set_test_timings(_timings)
def a():
pass
def b():
pass
yappi.start()
a()
yappi.stop()
yappi.start()
b()
yappi.stop()
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, 'a')
fsb = utils.find_stat_by_name(stats, 'b')
self.assertTrue(fsa is not None)
self.assertTrue(fsb is not None)
self.assertEqual(fsa.ttot, 1)
self.assertEqual(fsb.ttot, 1)
def test_default_context_name_cbk(self):
# Set context backend to configure default callbacks
yappi.set_context_backend("greenlet")
def a():
burn_cpu(0.1)
class GreenletA(gevent.Greenlet):
pass
yappi.start()
g = GreenletA(a)
g.start()
g.get()
yappi.stop()
gstats = yappi.get_greenlet_stats()
r2 = '''
GreenletA 3 0.100060 1
greenlet 1 0.000240 2
'''
self.assert_ctx_stats_almost_equal(r2, gstats)
async def main():
import yappi
yappi.set_clock_type("WALL")
yappi.start()
start = time.time()
task1 = asyncio.ensure_future(send_requests_poller())
task2 = asyncio.ensure_future(read_replies_poller())
await task1
await task2
finish(start)
yappi.get_func_stats().print_all(
columns={
0: ("name", 100),
1: ("ncall", 10),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8)
})
yappi.stop()
def getCpuProfile(self, profileDurationInSec):
try:
import yappi
except ImportError:
# Fallback (but log) if people call this method and we
# don't have yappi
self.logger.warning('getCpuProfile called without yappi installed')
return ''
# We need to lock this since it muck about with the global python
# profile hooks.
with self._profile_lock:
yappi.start()
time.sleep(profileDurationInSec)
yappi.stop()
stats = yappi.get_func_stats()
# Save the "pretty" output to a buffer and return the raw string
# Alternatively, we should return this as JSON and let the caller
# render it appropriately, but this is fine for now.
sio = StringIO()
stats.print_all(out=sio)
return sio.getvalue()
def wrapped(*args, **kwargs):
import yappi
yappi.start()
try:
return f(*args, **kwargs)
finally:
result = list(yappi.get_func_stats())
yappi.stop()
yappi.clear_stats()
result = [l for l in result if all([s not in l.full_name for s in skipped_lines])]
entries = result[:lines]
prefix = LOCALSTACK_ROOT_FOLDER
result = []
result.append('ncall\tttot\ttsub\ttavg\tname')
def c(num):
return str(num)[:7]
for e in entries:
name = e.full_name.replace(prefix, '')
result.append('%s\t%s\t%s\t%s\t%s' % (c(e.ncall), c(e.ttot), c(e.tsub), c(e.tavg), name))
result = '\n'.join(result)
print(result)
def test_callback(self):
self.context_id = 0
self.context_name = 'a'
yappi.set_context_id_callback(lambda: self.context_id)
yappi.set_context_name_callback(lambda: self.context_name)
yappi.start()
a()
self.context_id = 1
self.context_name = 'b'
a()
# Re-schedule context 0.
self.context_id = 0
self.context_name = 'a'
a()
yappi.stop()
threadstats = yappi.get_thread_stats().sort('name', 'ascending')
self.assertEqual(2, len(threadstats))
self.assertEqual(0, threadstats[0].id)
self.assertEqual('a', threadstats[0].name)
self.assertEqual(1, threadstats[1].id)
self.assertEqual('b', threadstats[1].name)
def test_app(self):
"""An 'integration' tests which profiles HTTP handling.
Note that pyproject.toml is configured to *not* run integration tests by
default.
To run integration tests from the command line:
pytest -m integration
"""
yappi.set_clock_type("wall")
yappi.start()
# Run TestClient in a with block to run startup / shutdown handlers
with TestClient(app=app) as client:
for _ in range(20):
resp = client.get("/")
assert resp.status_code == 200
yappi.stop()
with open("./out.txt", "w") as f:
# Filter to *just* local names. You cold also filter based on module
# name). Anything in the `YFuncStat` object.
yappi.get_func_stats(
filter_callback=lambda x: "python-examples/fastapi" in x.
full_name).print_all(
out=f,
columns={
0: ("name",
200), # The default is 36, which is way too short.
1: ("ncall", 5),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8),
},
)
def test_recursive_coroutine(self):
def a(n):
if n <= 0:
return
gevent.sleep(0.1)
burn_cpu(0.1)
g1 = gevent.spawn(a, n - 1)
g1.get()
g2 = gevent.spawn(a, n - 2)
g2.get()
yappi.set_clock_type("cpu")
yappi.start()
g = gevent.spawn(a, 3)
g.get() # run until complete, report exception (if any)
yappi.stop()
r1 = '''
..p/yappi/tests/test_asyncio.py:11 a 9/1 0.000124 0.400667 0.044519
../yappi/tests/utils.py:126 burn_cpu 4 0.000000 0.400099 0.100025
async_sleep 4 0.000000 0.000444 0.000111
'''
stats = yappi.get_func_stats()
self.assert_traces_almost_equal(r1, stats)
async def on_message(message):
BOT_PREFIX = "+"
if getenv("PRODUCTION") is None:
BOT_PREFIX = "-"
if not message.content.startswith(BOT_PREFIX):
return
# Split input
args = message.content[len(BOT_PREFIX):].split()
if len(args) == 0:
return
cmd = args[0].lower()
# the arg array ex. ["hello", "world"]
args = args[1:]
s = message.channel.send
if (cmd in {
"8",
"report",
"define",
"stars",
"homepage",
"clapify",
"cookie",
"say",
} and Preconditions.args_are_valid(args)):
return await s(embed=EmbedUtil.prep(
"That command expected an argument (or arguments), but you didn't give it any!",
"[Read the docs?](https://cakebot.club/docs/commands/)",
))
tcu_result = TextCommandsUtil.handle_common_commands(args, cmd)
if tcu_result != "":
return await s(tcu_result)
if cmd == "help":
return await s(embed=EmbedUtil.prep(
title="Help",
description=
"You can check out [this page of our website](https://cakebot.club/docs/commands/) for a full command list!",
))
elif cmd == "ping":
return await s(f"ЪЈЊ - websocket responded in {client.latency}")
elif cmd == "invite":
return await s(embed=EmbedUtil.prep(
"Invite Cakebot",
f"[Click here to invite me!]({oauth_url(580573141898887199, permissions=discord.Permissions.all())})",
))
elif cmd == "info":
return await s(embed=EmbedUtil.prep(
"Server Info",
TextCommandsUtil.data_template.format(
message.guild.name,
str(message.guild.owner),
len(message.guild.members),
message.guild.region,
message.guild.id,
message.guild.premium_subscription_count,
str(message.guild.is_icon_animated()),
str(message.guild.created_at),
str(message.guild.large),
str(message.guild.mfa_level == 1),
),
))
elif cmd == "report":
return await GitHubUtil.report(s, g, args, message)
elif cmd == "iss":
m = await s("Calculating...")
imp = IssApi.IssLocater()
lat = imp.lat
lon = imp.lon
from reverse_geocoder import search
geodata = search((lat, lon))
location = "{0}, {1}".format(geodata[0]["admin1"], geodata[0]["cc"])
await m.delete()
return await s(embed=EmbedUtil.prep(
"International Space Station", "Where it is right now!").add_field(
name="Location above Earth", value=str(location),
inline=False).add_field(
name="Latitude", value=str(lat), inline=False).add_field(
name="Longitude", value=str(lon), inline=False))
elif cmd == "fact":
return await s(embed=EmbedUtil.prep("Random Fact", FactImp().fact()))
elif cmd == "slots":
slotz = result()
top = row()
btm = row()
form = "win" if slotz[0] == 1 else "lose"
return await s(
f"Рађ{top[0]}{top[1]}{top[2]}\n"
# the line above contains unicode, DO NOT REMOVE
+ f"**>** {slotz[1][0]}{slotz[1][1]}{slotz[1][2]} **<**\n" +
f" {btm[0]}{btm[1]}{btm[2]}" + f"\n**You {form}!**")
elif cmd == "reboot":
if message.author.id in UserUtil.admins():
await s("Restarting. This may take up to 5 minutes.")
# make the bot crash, forcing our server to turn it back on
_exit(1)
else:
return await s(":x: **You are not authorized to run this!**")
elif cmd == "stars":
try:
return await s(
f"`{args[0]}` has *{g.get_repo(args[0]).stargazers_count}* stars."
)
except:
return await s(
"Failed to get count. Is the repository valid and public?")
elif cmd == "homepage":
try:
url = g.get_repo(args[0]).homepage
if url is None:
url = "(error: homepage not specified by owner)"
return await s(f"{args[0]}'s homepage is located at {url}")
except:
return await s(
"Failed to fetch homepage. Is the repository valid and public?"
)
elif cmd == "boomer":
return await s(file=discord.File("content/boomer.jpeg"))
elif cmd == "cookie" or cmd == "cookies":
subcommand = args[0]
args = args[1:]
userId = TextCommandsUtil.get_mentioned_id(args)
if subcommand in ["balance", "bal"]:
count = 0
if userId == 0:
# assume user wants themself
count = Database.get_count(message.author.id, config)
else:
count = Database.get_count(userId, config)
return await s(embed=EmbedUtil.prep(
title="Cookies",
description=f"User has {count} cookies.",
))
elif subcommand in ["give", "to"]:
if userId == 0:
return await s(
"I don't see who I should give the cookie to. Try mentioning them."
)
new_count = Database.add_cookie(userId, config)
return await s(
f"Gave <@!{userId}> a cookie. They now have {new_count} cookies."
)
elif cmd == "define":
if wordsapi_token is None:
return await s(
"This command is disabled due to a configuration error on my host's end - didn't find a WordsAPI token in the config!"
)
return await s(embed=TextCommandsUtil.define(args, wordsapi_token))
elif cmd == "start-profiler":
if message.author.id in UserUtil.admins():
await s(
"Started the profiler. Once you are done, run stop-profiler.")
yappi.set_clock_type("wall")
yappi.start()
else:
return await s(":x: **You are not authorized to run this!**")
elif cmd == "stop-profiler":
if message.author.id in UserUtil.admins():
await s("Saved profiler results to `profile.txt`.")
yappi.stop()
yappi.get_func_stats().print_all(open("profile.txt", "w"))
else:
return await s(":x: **You are not authorized to run this!**")
return model_fn(input_tensor)
tf.config.list_physical_devices('GPU')
model_name = 'diypilot_v9_small_FC_epoch_3'
loaded_model = keras.models.load_model(
'/home/jetson/diypilot/Jetson/autopilot/record/trained_models/' +
model_name + '/trt/')
frame = cv2.imread('/home/jetson/diypilot/notebooks/frame1.jpg')
frame_crop = crop_to_roi(frame)
yappi.start()
for i in range(1000):
start = time.time()
predictions = run_inference_for_single_image(loaded_model, frame_crop)
a = tf.make_tensor_proto(predictions['dense_1'])
predicted_swa = tf.make_ndarray(a)[0][0] * 90
#print()
end = time.time()
print("Inference took {}s".format(end - start))
frame_crop = cv2.flip(frame_crop, 0)
yappi.stop()
# retrieve thread stats by their thread id (given by yappi)
threads = yappi.get_thread_stats()
for thread in threads:
print("Function stats for (%s) (%d)" % (thread.name, thread.id))
yappi.get_func_stats(ctx_id=thread.id).print_all()
if options.verbose:
reprocess.setVerboseDebugOn()
keys = [ os.path.abspath(onefile) for onefile in args if os.path.exists(onefile)]
keys.sort()
working_dir = "analyse-modeling-%s" % time.strftime("%Y%m%d-%H%M%S")
base_dir = os.getcwd()
os.makedirs(working_dir)
os.chdir(working_dir)
for run in keys:
xsd = XSDataInputSaxsAnalysisModeling(scatterCurve=XSDataFile(XSDataString(run)), graphFormat=XSDataString("png"))
reprocess.startJob(xsd)
print("All %i jobs queued after %.3fs" % (len(args), time.time() - reprocess.startTime))
reprocess.join()
if yappi: yappi.stop()
print("All %i jobs processed after %.3fs" % (len(args), time.time() - reprocess.startTime))
print reprocess.statistics()
if yappi:
stat = yappi.get_stats(sort_type=yappi.SORTTYPE_TTOT)
res = {}
for i in stat.func_stats:
if i[0] in res:
res[i[0]][0] += i[1]
res[i[0]][1] += i[2]
else:
res[i[0]] = [i[1], i[2]]
keys = res.keys()
keys.sort(sortn)
with open("yappi.out", "w") as f:
f.write("ncall\t\ttotal\t\tpercall\t\tfunction%s" % (os.linesep))
def stop(self):
yappi.stop()
return '<a href="stats">stats</a>'
dep3_DEC_r = dep2_delta1 * (np.pi / 180)
dep3_arg1 = np.cos(dep3_DEC_r) - dep3_eta * np.sin(dep3_DEC_r)
dep3_arg2 = np.arctan(dep3_xi / dep3_arg1)
dep3_alpha1 = dep2_alpha1 + (180 / np.pi) * dep3_arg2
dep3_arg3 = np.sin(dep3_arg2)
dep3_arg4 = dep3_eta * np.cos(dep3_DEC_r) + np.sin(dep3_DEC_r)
dep3_delta1 = (180 / np.pi) * np.arctan((dep3_arg3 * dep3_arg4) / dep3_xi)
print '-' * 20
print 'RA =', dep3_alpha1
print 'DEC =', dep3_delta1
print '-' * 20
time5 = time.time()
print '-.-' * 20
print 'Processing time:'
print '- Imports: ', time2 - time1, 'seconds.'
print '- Picture: ', time3 - time2, 'seconds.'
print '- SExtractor: ', time4 - time3, 'seconds.'
print '- Match routines:', time5 - time4, 'seconds.'
print '- Total time: ', time5 - time1, 'seconds.'
print '-.-' * 20
yappi.start()
mainStarTracker()
yappi.get_func_stats().print_all() # Mostramos las stats de ejecucion
yappi.get_thread_stats().print_all()
print 'Mem usage: ' + str(yappi.get_mem_usage())
yappi.stop() # Paramos la monitorizacion de yappi
def stop_profiling():
"""Stop the profiler."""
# DELETE /profiler
yappi.stop()
