def test_pause_resume(self): yappi.set_context_id_callback(lambda: self.context_id) yappi.set_clock_type('wall') # Start in context 0. self.context_id = 0 yappi.start() time.sleep(0.08) # Switch to context 1. self.context_id = 1 time.sleep(0.05) # Switch back to context 0. self.context_id = 0 time.sleep(0.07) yappi.stop() t_stats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(2, len(t_stats)) self.assertEqual(0, t_stats[0].id) self.assertEqual(2, t_stats[0].sched_count) self.assertTrue(0.15 < t_stats[0].ttot < 0.3) self.assertEqual(1, t_stats[1].id) self.assertEqual(1, t_stats[1].sched_count) # Context 1 was first scheduled 0.08 sec after context 0. self.assertTrue(0.1 < t_stats[1].ttot < 0.2 )
def setUp(self): # reset everything back to default yappi.stop() yappi.clear_stats() yappi.set_clock_type('cpu') # reset to default clock type yappi.set_context_id_callback(None) yappi.set_context_name_callback(None)
def profile_with_yappi(func): import yappi yappi.set_clock_type("cpu") @functools.wraps(func) def wrapper(*args, **kwargs): # Profile function yappi.start(builtins=True) t1, t2 = timer1(), timer2() result = func(*args, **kwargs) t1, t2 = timer1() - t1, timer2() - t2 func_stats = yappi.get_func_stats() thread_stats = yappi.get_thread_stats() # Print stats print( f"\n\n{func.__name__}:\n\n" f'{"": <9}{timer1.__name__}: {t1:.9f}s\n' f'{"": <9}{timer2.__name__}: {t2:.9f}s' ) func_stats.print_all() thread_stats.print_all() return result return wrapper
def profile_threaded(filename): import yappi # noqa # yappi is not a dependency yappi.set_clock_type("cpu") try: import gil_load # noqa # same gil_load.init() gil_load.start(av_sample_interval=0.1, output_interval=10, output=sys.stdout) monitoring_gil = True except (RuntimeError, ImportError): monitoring_gil = False pass yappi.start() yield yappi.stop() if monitoring_gil: gil_load.stop() stats = gil_load.get() print("GIL load information: ", gil_load.format(stats)) p = yappi.get_func_stats() p = yappi.convert2pstats(p) p.dump_stats(filename) yappi.clear_stats()
def _start_profiling(self, clock): with lock: if yappi.is_running(): raise HTTPBadRequest("profile is already running") log.info("Starting profiling using %r clock", clock) yappi.set_clock_type(clock) yappi.start(builtins=True, profile_threads=True)
def run_stratrunner(strategy, exchange, symbol, timeframe, volume, profile): strat_dir_str = "noobit_user.strategies" strat_file_path = f"{strat_dir_str}.{strategy}" strategy = import_module(strat_file_path) # in every strategy file the class needs to be called "Strategy" strat = strategy.Strategy(exchange=exchange, symbol=symbol.upper(), timeframe=timeframe, volume=volume) runner = StratRunner(strats=[strat]) if profile: print("Profiling code with Yappi") yappi.set_clock_type("WALL") with yappi.run(): runner.run() # yappi.get_func_stats().print_all() stats = yappi.get_func_stats.sort(sort_type='totaltime', sort_order='desc') # returns all stats with sorting applied yappi.print_all(stats, sys.stdout, limit=10) else: runner.run()
def test_builtin_profiling(self): import threading def a(): import time time.sleep(0.4) # is a builtin function yappi.set_clock_type('wall') yappi.start(builtins=True) a() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'sleep') self.assertTrue(fsa is not None) self.assertTrue(fsa.ttot > 0.3) yappi.stop() yappi.clear_stats() def a(): pass yappi.start() t = threading.Thread(target=a) t.start() t.join() stats = yappi.get_func_stats()
def test_merge_load_different_clock_types(self): import threading yappi.start(builtins=True) def a(): b() def b(): c() def c(): pass t = threading.Thread(target=a) t.start() t.join() yappi.get_func_stats().sort("name", "asc").save("ystats1.ys") yappi.stop() yappi.clear_stats() yappi.start(builtins=False) t = threading.Thread(target=a) t.start() t.join() yappi.get_func_stats().save("ystats2.ys") yappi.stop() self.assertRaises(_yappi.error, yappi.set_clock_type, "wall") yappi.clear_stats() yappi.set_clock_type("wall") yappi.start() t = threading.Thread(target=a) t.start() t.join() yappi.get_func_stats().save("ystats3.ys") self.assertRaises(yappi.YappiError, yappi.YFuncStats().add("ystats1.ys").add, "ystats3.ys") stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys"]).sort("name") fsa = utils.find_stat_by_name(stats, "a") fsb = utils.find_stat_by_name(stats, "b") fsc = utils.find_stat_by_name(stats, "c") self.assertEqual(fsa.ncall, 2) self.assertEqual(fsa.ncall, fsb.ncall, fsc.ncall)
def init_yappi(): import atexit print('[YAPPI START]') yappi.set_clock_type('wall') yappi.start()
def yappi_profile(path: Union[Callable[[], str], str] = None, wall_clock: bool = True): try: import yappi # pylint: disable=import-error except ImportError: print("Failed to run profiler, yappi is not installed") yield return yappi.set_clock_type("wall" if wall_clock else "cpu") yappi.start() yield yappi.stop() # pylint:disable=no-member if path: stats = yappi.get_func_stats() fpath = path() if callable(path) else path stats.save(fpath, "callgrind") else: yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() yappi.clear_stats()
def main_loop(self): if self.args.profile == -1: profile_file = None else: profile_file = self.args.profile or ('mars_' + self.__class__.__name__ + '.prof') try: if profile_file: import yappi yappi.set_clock_type('wall') yappi.start(builtins=False, profile_threads=False) with self.pool: try: self.start_service() self._running = True while True: self.pool.join(1) for proc in self.pool.processes: if not proc.is_alive(): self.service_logger.fatal( 'Process %d exited unpredictably. exitcode=%d', proc.pid, proc.exitcode) raise KeyboardInterrupt except: self._running = False self.stop_service() finally: self._running = False if profile_file: import yappi yappi.logger = logging.getLogger(__name__) p = yappi.convert2pstats(yappi.get_func_stats()) p.strip_dirs() p.sort_stats('time') p.print_stats(40) p.dump_stats(profile_file)
def profile(filename="results", engine="yappi", clock="wall", output_type="pstat"): directory = Path("profiling/results/") directory.mkdir(parents=True, exist_ok=True) path = os.path.join(directory, f"{filename}.prof") if engine == "yappi": yappi.set_clock_type(clock) try: yappi.start(builtins=True, profile_threads=False) yield finally: yappi.stop() stats = yappi.get_func_stats() stats.print_all() stats.save(path, type=output_type) else: profile = cProfile.Profile() try: profile.enable() yield finally: profile.disable() profile.print_stats() profile.dump_stats(path)
def test_exception_raised(self): def a(n): burn_cpu(0.1) burn_io_gevent(0.1) if (n == 0): raise Exception a(n - 1) yappi.set_clock_type("cpu") yappi.start() try: gevent.spawn(a, 3).get() except Exception: pass yappi.stop() stats = yappi.get_func_stats() t1 = ''' tests/test_gevent.py:118 a 4/1 0.000149 0.400614 0.100153 ../yappi/tests/utils.py:126 burn_cpu 4 0.000000 0.400208 0.100052 ''' self.assert_traces_almost_equal(t1, stats)
def start(self): yappi.set_context_id_callback(lambda:id(greenlet.getcurrent())) yappi.set_context_name_callback(lambda:greenlet.getcurrent().__class__.__name__) yappi.set_clock_type('cpu') yappi.start(builtins=True) self._isStart = True return 'success'
def setUp(self): super(GeventTest, self).setUp() yappi.set_clock_type("cpu") yappi.set_context_backend("greenlet") yappi.set_context_name_callback(self.get_greenlet_name) gevent.getcurrent().name = "Main" gevent.get_hub().name = "Hub"
def test_basic(self): yappi.set_clock_type('wall') def dummy(): pass def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') dummy() # call dummy to force ctx name to be retrieved again. self.assertTrue(tsa is not None) # TODO: I put dummy() to fix below, remove the comments after a while. self.assertTrue( # FIX: I see this fails sometimes? tsm is not None, 'Could not find "_MainThread". Found: %s' % (', '.join(utils.get_stat_names(tstats))))
def main(): try: resolution = float(os.getenv('DEBUG_BLOCKING')) import eventlet.debug eventlet.debug.hub_blocking_detection(state=True, resolution=resolution) except (ValueError, TypeError): pass try: import yappi as profiler profiler.set_clock_type('wall') profiler.start(builtins=True) except ImportError: profiler = None pass try: _main() except KeyboardInterrupt: pass if profiler: print("Stopping profiler") profiler.stop() stats = profiler.get_func_stats() stats.save('/tmp/profile.callgrind', type='callgrind') print("Done")
def test_pause_resume(self): yappi.set_context_id_callback(lambda: self.context_id) yappi.set_clock_type('wall') # Start in context 0. self.context_id = 0 yappi.start() time.sleep(0.08) # Switch to context 1. self.context_id = 1 time.sleep(0.05) # Switch back to context 0. self.context_id = 0 time.sleep(0.07) yappi.stop() t_stats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(2, len(t_stats)) self.assertEqual(0, t_stats[0].id) self.assertEqual(2, t_stats[0].sched_count) self.assertAlmostEqual(0.2, t_stats[0].ttot, places=2) self.assertEqual(1, t_stats[1].id) self.assertEqual(1, t_stats[1].sched_count) # Context 1 was first scheduled 0.08 sec after context 0. self.assertAlmostEqual(0.12, t_stats[1].ttot, places=2)
def test_pause_resume(self): yappi.set_context_id_callback(lambda: self.context_id) yappi.set_clock_type('wall') # Start in context 0. self.context_id = 0 yappi.start() time.sleep(0.08) # Switch to context 1. self.context_id = 1 time.sleep(0.05) # Switch back to context 0. self.context_id = 0 time.sleep(0.07) yappi.stop() t_stats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(2, len(t_stats)) self.assertEqual(0, t_stats[0].id) self.assertEqual(2, t_stats[0].sched_count) self.assertTrue(0.15 < t_stats[0].ttot < 0.7, t_stats[0].ttot) self.assertEqual(1, t_stats[1].id) self.assertEqual(1, t_stats[1].sched_count) # Context 1 was scheduled for 0.05 seconds during the run of the # profiler self.assert_almost_equal(t_stats[1].ttot, 0.05)
def test_singlethread_profiling(self): yappi.set_clock_type("wall") def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(profile_threads=False) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, "Worker1.a") fsa2 = utils.find_stat_by_name(stats, "a") self.assertTrue(fsa1 is None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa2.ttot > 0.1)
def test_basic(self): import threading import time yappi.set_clock_type('wall') def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') self.assertTrue(tsa is not None) self.assertTrue(tsm is not None) # FIX: I see this fails sometimes?
def profile_threaded(filename): import yappi # noqa # yappi is not a dependency import gil_load # noqa # same yappi.set_clock_type("cpu") try: gil_load.init() gil_load.start(av_sample_interval=0.1, output_interval=3, output=sys.stdout) monitoring_gil = True except RuntimeError: monitoring_gil = False pass yappi.start() yield yappi.stop() if monitoring_gil: gil_load.stop() print("Gil was held %0.1f %% of the time" % (100 * gil_load.get()[0])) p = yappi.get_func_stats() p = yappi.convert2pstats(p) p.dump_stats(filename) yappi.clear_stats()
def test_singlethread_profiling(self): import threading import time yappi.set_clock_type('wall') def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(profile_threads=False) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa2.ttot > 0.1)
def __init__(self, app, calls_to_track: Dict[str, Tuple[Callable]], max_profiler_mem: int = 50_000_000): for metric_name, profiled_functions in calls_to_track.items(): if len(metric_name) == 0: raise ValueError('A Server-Timing metric name cannot be empty') # https://httpwg.org/specs/rfc7230.html#rule.token.separators # USASCII (7 bits), only visible characters (no non-printables or space), no double-quote or delimiter if not metric_name.isascii( ) or not metric_name.isprintable() or re.search( r'[ "(),/:;<=>?@\[\\\]{}]', metric_name) is not None: raise ValueError( '"{}" contains an invalid character for a Server-Timing metric name' .format(metric_name)) if not all( inspect.isfunction(profiled) for profiled in profiled_functions): raise TypeError( 'One of the targeted functions for key "{}" is not a function' .format(metric_name)) self.app = app self.calls_to_track = { name: list(tracked_funcs) for name, tracked_funcs in calls_to_track.items() } self.max_profiler_mem = max_profiler_mem yappi.set_tag_callback(_get_context_tag) yappi.set_clock_type("wall") yappi.start()
def test_lambda(self): f = lambda: time.sleep(0.3) yappi.set_clock_type("wall") yappi.start() f() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, "<lambda>") self.assertTrue(fsa.ttot > 0.1)
def test_lambda(self): f = lambda: time.sleep(0.3) yappi.set_clock_type("wall") yappi.start() f() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, '<lambda>') self.assertTrue(fsa.ttot > 0.1)
def start(self): yappi.set_clock_type("WALL") if self.sync: yappi.start() context_manager = nullcontext() else: context_manager = yappi.run() return context_manager
def _start_profiling(self, clock): with lock: if yappi.is_running(): raise http.Error( http.BAD_REQUEST, "profile is already running") log.info("Starting profiling using %r clock", clock) yappi.set_clock_type(clock) yappi.start(builtins=True, profile_threads=True)
def _start( self, config, current_time ): yappi.clear_stats() clock = self._get_clock_type( config.profile_clock, self._allowed_clocks, self._profile_clock ) if clock != self._profile_clock: self._profile_clock = clock yappi.set_clock_type( self._profile_clock ) global_log.log( scalyr_logging.DEBUG_LEVEL_0, "Starting profiling using '%s' clock. Duration: %d seconds" % (self._profile_clock, self._profile_end - self._profile_start) ) yappi.start()
def _start_profiling(self, clock): with lock: if yappi.is_running(): raise http.Error(http.BAD_REQUEST, "profile is already running") log.info("Starting profiling using %r clock", clock) yappi.set_clock_type(clock) yappi.start(builtins=True, profile_threads=True)
def decorator(*args, **kwargs): filename = '{}/callgrind.{}.prof'.format(PATH, int(time.time() * 1000)) yappi.set_clock_type('cpu') yappi.start(builtins=True) result = func(*args, **kwargs) stats = yappi.get_func_stats() stats.save(filename, type='callgrind') return result
def start_profiling(): # This will show the performance impact of each function. # If you want to see the performance impact of each _line_ in a function, # pip install line_profiler # and add @profile annotations to functions of interest, then run kernprof # as described in the line_profiler package. import yappi yappi.set_clock_type('cpu') yappi.start() atexit.register(_dump_profiling_stats)
def test_basic_old_style(self): def a(): burn_io_gevent(0.1) burn_io(0.1) burn_io_gevent(0.1) burn_io(0.1) burn_io_gevent(0.1) burn_cpu(0.3) yappi.set_clock_type("wall") yappi.start(builtins=True) g1 = self.spawn_greenlet("a_1", a) g1.get() g2 = self.spawn_greenlet("a_2", a) g2.get() yappi.stop() r1 = ''' ..p/yappi/tests/test_asyncio.py:43 a 2 0.000118 1.604049 0.802024 burn_io_gevent 6 0.000000 0.603239 0.100540 ../yappi/tests/utils.py:126 burn_cpu 2 0.000000 0.600026 0.300013 ..p/yappi/tests/utils.py:135 burn_io 4 0.000025 0.400666 0.100166 ''' stats = yappi.get_func_stats() self.assert_traces_almost_equal(r1, stats) gstats = yappi.get_greenlet_stats() r2 = ''' Main 1 1.623057 3 Main/a_1 3 0.812399 1 Main/a_2 4 0.810234 1 ''' self.assert_ctx_stats_almost_equal(r2, gstats) yappi.clear_stats() yappi.set_clock_type("cpu") yappi.start(builtins=True) g1 = self.spawn_greenlet("a_1", a) g1.get() g2 = self.spawn_greenlet("a_2", a) g2.get() yappi.stop() stats = yappi.get_func_stats() r1 = ''' ..p/yappi/tests/test_asyncio.py:43 a 2 0.000117 0.601170 0.300585 ../yappi/tests/utils.py:126 burn_cpu 2 0.000000 0.600047 0.300024 burn_io_gevent 6 0.000159 0.000801 0.000134 ''' self.assert_traces_almost_equal(r1, stats) gstats = yappi.get_greenlet_stats() r2 = ''' Main/a_2 6 0.301190 1 Main/a_1 5 0.300960 1 Main 1 0.000447 3 ''' self.assert_ctx_stats_almost_equal(r2, gstats)
def clusterizacao(request, debate_id, qtdGrupos=3): ## COLOCAR COMO OPÇÃO PARA O USUÁRIO SE ELE QUER AGRUPAR ## PELO POSICIONAMENTO INICIAL OU FINAL print "view-clusterização em funcionamento!!!" inicio = datetime.now() print inicio, "view clusterizacao" yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() auxResult = clusterArgInicial(debate_id) duration = time.time() - start stats = yappi.get_func_stats() stats.save('clusterArgInicial.out', type='callgrind') tese = auxResult[5] yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() # resultado = gruposArgumentacao(auxResult, qtdeGrupos=3, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=4, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=5, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=6, LSA=True, Normalizacao=True) resultado = gruposArgumentacao(auxResult, qtdeGrupos=int(qtdGrupos), LSA=None, Normalizacao=True) duration = time.time() - start stats = yappi.get_func_stats() stats.save('gruposArgumentacao.out', type='callgrind') grupo1 = resultado[0] grupo2 = resultado[1] grupo3 = resultado[2] grupo4 = resultado[3] grupo5 = resultado[4] grupo6 = resultado[5] context = RequestContext(request,{'results' : [grupo1,grupo2,grupo3,grupo4,\ len(grupo1),len(grupo2),len(grupo3),len(grupo4), tese, \ grupo5, len(grupo5), grupo6, len(grupo6)], 'grupo' : Grupo.objects.filter(idgrupo=1064)[0]}) return render(request, 'clusterizacao.html', context)
def clusterizacao(request, debate_id, qtdGrupos=3): ## COLOCAR COMO OPÇÃO PARA O USUÁRIO SE ELE QUER AGRUPAR ## PELO POSICIONAMENTO INICIAL OU FINAL print "view-clusterização em funcionamento!!!" inicio = datetime.now() print inicio,"view clusterizacao" yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() auxResult = clusterArgInicial(debate_id) duration = time.time() - start stats = yappi.get_func_stats() stats.save('clusterArgInicial.out', type = 'callgrind') tese = auxResult[5] yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() # resultado = gruposArgumentacao(auxResult, qtdeGrupos=3, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=4, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=5, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=6, LSA=True, Normalizacao=True) resultado = gruposArgumentacao(auxResult, qtdeGrupos=int(qtdGrupos), LSA=None, Normalizacao=True) duration = time.time() - start stats = yappi.get_func_stats() stats.save('gruposArgumentacao.out', type = 'callgrind') grupo1 = resultado[0] grupo2 = resultado[1] grupo3 = resultado[2] grupo4 = resultado[3] grupo5 = resultado[4] grupo6 = resultado[5] context = RequestContext(request,{'results' : [grupo1,grupo2,grupo3,grupo4,\ len(grupo1),len(grupo2),len(grupo3),len(grupo4), tese, \ grupo5, len(grupo5), grupo6, len(grupo6)], 'grupo' : Grupo.objects.filter(idgrupo=1064)[0]}) return render(request, 'clusterizacao.html',context)
def process(fn): """Iteratively process an file object. Arguments: fn - file name """ if args.profile: if args.verbose: print("\x1b[6;30;43m[i]\x1b[0m profiling is on...") yappi.clear_stats() yappi.set_clock_type('cpu') yappi.start(builtins=True) db = initdb(fn) with open(os.path.join(args.input, fn), 'r') as fileobj: parser = ijson.common.items(ijson.parse(fileobj, multiple_values=True), '') if args.trace == 'camflow': if args.verbose: print( "\x1b[6;30;42m[+]\x1b[0m parsing file {} in CAMFLOW mode..." .format(fn)) ptj.parsecf(parser, db, fn) elif args.trace == 'darpa': if args.verbose: print( "\x1b[6;30;42m[+]\x1b[0m parsing file {} in DARPA mode...". format(fn)) ptj.parsedp(parser, db, fn) elif args.trace == 'cadets2' or args.trace == 'fivedirections': if args.verbose: print( "\x1b[6;30;42m[+]\x1b[0m parsing file {} in CADETS2/FIVEDIRECTIONS mode..." .format(fn)) ptj.parsecd(parser, db, fn) else: raise NotImplementedError( "cannot run traces from an unknown system") if args.profile: yappi.stop() if args.verbose: print("\x1b[6;30;43m[i]\x1b[0m profiling is done...") stat = yappi.get_func_stats() stat.save(fn + '.prof', type='callgrind') fileobj.close() return
def _start_profiling(clock, builtins, threads): global yappi logging.debug("Starting profiling") with _lock: import yappi # yappi start semantics are a bit too liberal, returning success if # yappi is already started, happily having too different code paths # that thinks they own the single process profiler. if yappi.is_running(): raise Error('Profiler is already running') yappi.set_clock_type(clock) yappi.start(builtins=builtins, profile_threads=threads)
def test_tagging_cputime(self): tlocal = threading.local() def tag_cbk(): try: return tlocal._tag except Exception as e: #print(e) return -1 def a(tag): tlocal._tag = tag burn_cpu(0.1) _TCOUNT = 5 ts = [] yappi.set_clock_type("cpu") tlocal._tag = 0 yappi.set_tag_callback(tag_cbk) yappi.start() for i in range(_TCOUNT): t = threading.Thread(target=a, args=(i + 1, )) ts.append(t) for t in ts: t.start() for t in ts: t.join() yappi.stop() traces = yappi.get_func_stats() t1 = ''' ..op/p/yappi/tests/test_tags.py:21 a 5 0.000137 0.500562 0.000000 ../yappi/tests/utils.py:125 burn_cpu 5 0.000000 0.500424 0.000000 ''' self.assert_traces_almost_equal(t1, traces) traces = yappi.get_func_stats(filter={'tag': 1}) t1 = ''' ../yappi/tests/utils.py:125 burn_cpu 1 0.000000 0.100125 0.100125 ''' self.assert_traces_almost_equal(t1, traces) traces = yappi.get_func_stats(filter={'tag': 3}) t1 = ''' ../yappi/tests/utils.py:125 burn_cpu 1 0.000000 0.100128 0.100128 ''' self.assert_traces_almost_equal(t1, traces)
def test_no_stats_different_clock_type_load(self): def a(): pass yappi.start() a() yappi.stop() yappi.get_func_stats().save("ystats1.ys") yappi.clear_stats() yappi.set_clock_type("WALL") yappi.start() yappi.stop() stats = yappi.get_func_stats().add("ystats1.ys") fsa = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa is not None)
def test_get_clock(self): yappi.set_clock_type('cpu') clock_info = yappi.get_clock() self.assertEqual('cpu', clock_info['type']) self.assertTrue('api' in clock_info) self.assertTrue('resolution' in clock_info) yappi.set_clock_type('wall') clock_info = yappi.get_clock() self.assertEqual('wall', clock_info['type']) time.sleep(0.1) duration = yappi.get_clock()['time'] - clock_info['time'] self.assertAlmostEqual(0.1, duration, places=2)
def test_get_clock(self): yappi.set_clock_type('cpu') self.assertEqual('cpu', yappi.get_clock_type()) clock_info = yappi.get_clock_info() self.assertTrue('api' in clock_info) self.assertTrue('resolution' in clock_info) yappi.set_clock_type('wall') self.assertEqual('wall', yappi.get_clock_type()) t0 = yappi.get_clock_time() time.sleep(0.1) duration = yappi.get_clock_time() - t0 self.assertAlmostEqual(0.1, duration, places=2)
def test_module_stress(self): self.assertEqual(yappi.is_running(), False) yappi.start() yappi.clear_stats() self.assertRaises(_yappi.error, yappi.set_clock_type, "wall") yappi.stop() yappi.clear_stats() yappi.set_clock_type("cpu") self.assertRaises(yappi.YappiError, yappi.set_clock_type, "dummy") self.assertEqual(yappi.is_running(), False) yappi.clear_stats() yappi.clear_stats()
def test_get_clock(self): yappi.set_clock_type("cpu") self.assertEqual("cpu", yappi.get_clock_type()) clock_info = yappi.get_clock_info() self.assertTrue("api" in clock_info) self.assertTrue("resolution" in clock_info) yappi.set_clock_type("wall") self.assertEqual("wall", yappi.get_clock_type()) t0 = yappi.get_clock_time() time.sleep(0.1) duration = yappi.get_clock_time() - t0 self.assertTrue(0.05 < duration < 0.2)
def start(self): # Lazy import so we do not effect runtime environment if profiling is # not used. global yappi import yappi # yappi start semantics are a bit too liberal, returning success if # yappi is already started, happily having two different code paths # that thinks they own the single process profiler. if yappi.is_running(): raise UsageError('CPU profiler is already running') logging.info("Starting CPU profiling") yappi.set_clock_type(self.clock) yappi.start(builtins=self.builtins, profile_threads=self.threads)
def main(args): options, args = parse_args(args) if options.monkeypatch: if options.monkeypatch == "cthreading": import cthreading cthreading.monkeypatch() elif options.monkeypatch == "pthreading": import pthreading pthreading.monkey_patch() else: raise ValueError("Usupported monkeypatch %r" % options.monkeypatch) try: import Queue as queue _range = xrange except ImportError: import queue _range = range import threading if options.profile: import yappi yappi.set_clock_type('cpu') yappi.start(builtins=True, profile_threads=True) leftmost = queue.Queue() left = leftmost for i in _range(options.whispers): right = queue.Queue() t = threading.Thread(target=whisper, args=(left, right)) t.daemon = True t.start() left = right for i in _range(options.jobs): right.put(1) for i in _range(options.jobs): n = leftmost.get() assert n == options.whispers + 1 if options.profile: yappi.stop() stats = yappi.get_func_stats() stats.save(options.profile, 'pstat')
def main(profiler, clock_type, concurrency): outfile = "callgrind.mitmdump-{}-c{}".format(clock_type, concurrency) a = ApacheBenchThread(concurrency) a.start() if profiler == "yappi": yappi.set_clock_type(clock_type) yappi.start(builtins=True) print("Start mitmdump...") mitmdump(["-k", "-q", "-S", "1024example"]) print("mitmdump stopped.") print("Save profile information...") if profiler == "yappi": yappi.stop() stats = yappi.get_func_stats() stats.save(outfile, type='callgrind') print("Done.")
def test_builtin_profiling(self): def a(): time.sleep(0.4) # is a builtin function yappi.set_clock_type('wall') yappi.start(builtins=True) a() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'sleep') self.assertTrue(fsa is not None) self.assertTrue(fsa.ttot > 0.3) yappi.stop() yappi.clear_stats() def a(): pass yappi.start() t = threading.Thread(target=a) t.start() t.join() stats = yappi.get_func_stats()
def run(func, options): if options.monkeypatch: if options.monkeypatch == "cthreading": import cthreading cthreading.monkeypatch() elif options.monkeypatch == "pthreading": import pthreading pthreading.monkey_patch() else: raise ValueError("Usupported monkeypatch %r" % options.monkeypatch) if options.profile: import yappi yappi.set_clock_type('cpu') yappi.start(builtins=True, profile_threads=True) func(options) if options.profile: yappi.stop() stats = yappi.get_func_stats() stats.save(options.profile, 'pstat')
def _capture_profile(fname=''): if not fname: yappi.set_clock_type('cpu') # We need to set context to greenlet to profile greenlets # https://bitbucket.org/sumerc/yappi/pull-requests/3 yappi.set_context_id_callback( lambda: id(greenlet.getcurrent())) yappi.set_context_name_callback( lambda: greenlet.getcurrent().__class__.__name__) yappi.start() else: yappi.stop() stats = yappi.get_func_stats() # User should provide filename. This file with a suffix .prof # will be created in temp directory. try: stats_file = os.path.join(tempfile.gettempdir(), fname + '.prof') stats.save(stats_file, "pstat") except Exception as e: print("Error while saving the trace stats ", str(e)) finally: yappi.clear_stats()
def enable(self): if self.wall: yappi.clear_stats() yappi.set_clock_type("wall") yappi.start()
def setUp(self): if yappi.is_running(): yappi.stop() yappi.clear_stats() yappi.set_clock_type('cpu') # reset to default clock type
def main(argv): parser = argparse.ArgumentParser(description='Anonymous Tunnel CLI interface') try: parser.add_argument('-p', '--socks5', help='Socks5 port') parser.add_argument('-x', '--exit', help='Allow being an exit-node') parser.add_argument('-i', '--introduce', help='Introduce the dispersy port of another tribler instance') parser.add_argument('-d', '--dispersy', help='Dispersy port') parser.add_argument('-c', '--crawl', help='Enable crawler and use the keypair specified in the given filename') parser.add_argument('-j', '--json', help='Enable JSON api, which will run on the provided port number ' + '(only available if the crawler is enabled)', type=int) parser.add_argument('-y', '--yappi', help="Profiling mode, either 'wall' or 'cpu'") parser.add_help = True args = parser.parse_args(sys.argv[1:]) except argparse.ArgumentError: parser.print_help() sys.exit(2) socks5_port = int(args.socks5) if args.socks5 else None introduce_port = int(args.introduce) if args.introduce else None dispersy_port = int(args.dispersy) if args.dispersy else -1 crawl_keypair_filename = args.crawl profile = args.yappi if args.yappi in ['wall', 'cpu'] else None if profile: yappi.set_clock_type(profile) yappi.start(builtins=True) logger.error("Profiling using %s time" % yappi.get_clock_type()['type']) if crawl_keypair_filename and not os.path.exists(crawl_keypair_filename): logger.error("Could not find keypair filename", crawl_keypair_filename) sys.exit(1) settings = TunnelSettings() # For disbling anonymous downloading, limiting download to hidden services only settings.min_circuits = 0 settings.max_circuits = 0 if socks5_port is not None: settings.socks_listen_ports = range(socks5_port, socks5_port + 5) else: settings.socks_listen_ports = [random.randint(1000, 65535) for _ in range(5)] settings.become_exitnode = True if args.exit in ['true'] else False if settings.become_exitnode: logger.info("Exit-node enabled") else: logger.info("Exit-node disabled") tunnel = Tunnel(settings, crawl_keypair_filename, dispersy_port) StandardIO(LineHandler(tunnel, profile)) tunnel.start(introduce_port) if crawl_keypair_filename and args.json > 0: cherrypy.config.update({'server.socket_host': '0.0.0.0', 'server.socket_port': args.json}) cherrypy.quickstart(tunnel) else: while True: time.sleep(1)
def filter_opencv(im): for i in range(100000): im += 1 return im def filter_homemade(im): for i in range(100000): im += 2 return im import yappi filename = 'callgrind.filename.prof' yappi.set_clock_type('cpu') yappi.start(builtins=True) filter_homemade(0) filter_opencv(0) stats = yappi.get_func_stats() stats.save(filename, type='callgrind')
def test_ctx_stats(self): from threading import Thread DUMMY_WORKER_COUNT = 5 yappi.start() class DummyThread(Thread): pass def dummy(): pass def dummy_worker(): pass for i in range(DUMMY_WORKER_COUNT): t = DummyThread(target=dummy_worker) t.start() t.join() yappi.stop() stats = yappi.get_thread_stats() tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is not None) yappi.clear_stats() time.sleep(1.0) _timings = {"a_1": 6, "b_1": 5, "c_1": 3, "d_1": 1, "a_2": 4, "b_2": 3, "c_2": 2, "d_2": 1} _yappi._set_test_timings(_timings) class Thread1(Thread): pass class Thread2(Thread): pass def a(): b() def b(): c() def c(): d() def d(): time.sleep(0.6) yappi.set_clock_type("wall") yappi.start() t1 = Thread1(target=a) t1.start() t2 = Thread2(target=a) t2.start() t1.join() t2.join() stats = yappi.get_thread_stats() # the fist clear_stats clears the context table? tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is None) tst1 = utils.find_stat_by_name(stats, "Thread1") tst2 = utils.find_stat_by_name(stats, "Thread2") tsmain = utils.find_stat_by_name(stats, "_MainThread") dummy() # call dummy to force ctx name to be retrieved again. self.assertTrue(len(stats) == 3) self.assertTrue(tst1 is not None) self.assertTrue(tst2 is not None) # TODO: I put dummy() to fix below, remove the comments after a while. self.assertTrue( # FIX: I see this fails sometimes tsmain is not None, 'Could not find "_MainThread". Found: %s' % (", ".join(utils.get_stat_names(stats))) ) self.assertTrue(1.0 > tst2.ttot >= 0.5) self.assertTrue(1.0 > tst1.ttot >= 0.5) # test sorting of the ctx stats stats = stats.sort("totaltime", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot >= stat.ttot) prev_stat = stat stats = stats.sort("totaltime", "asc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot <= stat.ttot) prev_stat = stat stats = stats.sort("schedcount", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.sched_count >= stat.sched_count) prev_stat = stat stats = stats.sort("name", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.name >= stat.name) prev_stat = stat self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sorttype_arg") self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sortorder_arg")
def normalizacaoWordnet(listaAdjetivos, listaSubstantivos, listaVerbos,listaOutros, st_tagcomAce_posInicial): ################################################################ ### MEDIÇÃO DE PROCESSAMENTO / DESEMPENHO / REQUISIÕES ######### ################################################################ yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() ######################################################################################### ### BUSCA PELOS TERMOS SINÔNIMOS EM st_WordNet E MONTA O DICIONÁRIO COM AS RELAÇÕES ## ######################################################################################### qtdeTermosTotal = 0 dicionario = {} ##guarda todas as relações de sinonímia para os termos de cada argumentação #print "BUSCA PELOS TERMOS SINÔNIMOS EM st_WordNet E MONTA O DICIONÁRIO COM AS RELAÇÕES" for iST in range(len(st_tagcomAce_posInicial)): qtdeTermos = 0 for jST in range(len(st_tagcomAce_posInicial[iST])): qtdeTermos = qtdeTermos + 1 radical = st_tagcomAce_posInicial[iST][jST][0] #termo reduzido ao seu radical de formação (aplicação de stemmer - RSLP) etiqueta = st_tagcomAce_posInicial[iST][jST][1] #etiqueta morfológica do termo com base no Tagger NPLNet # print "etiqueta", etiqueta if etiqueta == "VAUX" or etiqueta == "PCP": etiqueta = "V" # print "etiqueta", etiqueta chave = radical, etiqueta # print chave if etiqueta == "N": listSub = [] for i in range(len(listaSubstantivos)): for aux_radical in listaSubstantivos[i][2]: if aux_radical == radical: listSub.append(listaSubstantivos[i][2]) dicionario[chave] = listSub elif etiqueta == "V": listVerb = [] for i in range(len(listaVerbos)): for aux_radical in listaVerbos[i][2]: if aux_radical == radical: listVerb.append(listaVerbos[i][2]) dicionario[chave] = listVerb elif etiqueta == "ADJ": listAdj = [] for i in range(len(listaAdjetivos)): for aux_radical in listaAdjetivos[i][2]: if aux_radical == radical: listAdj.append(listaAdjetivos[i][2]) dicionario[chave] = listAdj else: listOutros = [] for i in range(len(listaOutros)): for aux_radical in listaOutros[i][2]: if aux_radical == radical: listOutros.append(listaOutros[i][2]) dicionario[chave] = listOutros qtdeTermosTotal = qtdeTermosTotal + qtdeTermos # print "dicionario" # pprint(dicionario) ### COLOCANDO TODOS OS SINONIMOS ENCONTRADOS NUMA UNICA LISTA! ### EXCLUINDO TERMOS SINONIMOS REPETIDOS DO DICIONARIO for chave, val in dicionario.iteritems(): auxDic = [] for iV in range(len(val)): for iVa in val[iV]: if iVa not in auxDic: auxDic.append(iVa) dicionario[chave] = auxDic ######################################################################################### ### REALIZA A TROCA DO TERMOS SINÔNIMOS POR UM ÚNICO TERMO E MONTA OS NOVOS ## ### POSICIONAMENTOS INICIAIS PARA ANÁLISE DE SIMILARIDADE NA VARIÁVEL norm_porInicial ## ######################################################################################### norm_posInicial = [] listaTermos = [] listAux = [] cont = 0 for idST in range(len(st_tagcomAce_posInicial)): for tupla in st_tagcomAce_posInicial[idST]: termoStr = tupla[0] etiquetaStr = tupla[1] if etiquetaStr == "PCP" or etiquetaStr == "VAUX": etiquetaStr = "V" tupla = termoStr, etiquetaStr if cont == 0: inicializarLista(tupla, listaTermos) inicializarLista(termoStr, listAux) cont = cont + 1 else: elemento = verificarDicionario(tupla, listaTermos, dicionario) if elemento == False: inserir(tupla, listaTermos) inserir(termoStr, listAux) else: inserir(elemento, listAux) norm_posInicial.append(listAux) listAux = limparLista(listAux) ################################################################ ### MEDIÇÃO DE PROCESSAMENTO / DESEMPENHO / REQUISIÕES ######### ################################################################ duration = time.time() - start stats = yappi.get_func_stats() stats.save('normalizacaoWordnet.out', type = 'callgrind') ######################################################################################### return norm_posInicial
def test_ctx_stats(self): from threading import Thread DUMMY_WORKER_COUNT = 5 yappi.start() class DummyThread(Thread): pass def dummy_worker(): pass for i in range(DUMMY_WORKER_COUNT): t = DummyThread(target=dummy_worker) t.start() t.join() yappi.stop() stats = yappi.get_thread_stats() tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is not None) yappi.clear_stats() import time time.sleep(1.0) _timings = {"a_1":6,"b_1":5,"c_1":3, "d_1":1, "a_2":4,"b_2":3,"c_2":2, "d_2":1} _yappi._set_test_timings(_timings) class Thread1(Thread): pass class Thread2(Thread): pass def a(): b() def b(): c() def c(): d() def d(): time.sleep(0.6) yappi.set_clock_type("wall") yappi.start() t1 = Thread1(target=a) t1.start() t2 = Thread2(target=a) t2.start() t1.join() t2.join() stats = yappi.get_thread_stats() # the fist clear_stats clears the context table? tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is None) tst1 = utils.find_stat_by_name(stats, "Thread1") tst2 = utils.find_stat_by_name(stats, "Thread2") tsmain = utils.find_stat_by_name(stats, "_MainThread") #stats.print_all() self.assertTrue(len(stats) == 3) self.assertTrue(tst1 is not None) self.assertTrue(tst2 is not None) self.assertTrue(tsmain is not None) # I see this fails sometimes, probably # because Py_ImportNoBlock() fails to import and get the thread class name # sometimes. self.assertTrue(1.0 > tst2.ttot >= 0.5) self.assertTrue(1.0 > tst1.ttot >= 0.5) # test sorting of the ctx stats stats = stats.sort("totaltime", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot >= stat.ttot) prev_stat = stat stats = stats.sort("totaltime", "asc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot <= stat.ttot) prev_stat = stat stats = stats.sort("schedcount", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.sched_count >= stat.sched_count) prev_stat = stat stats = stats.sort("name", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.name >= stat.name) prev_stat = stat self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sorttype_arg") self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sortorder_arg")
fname = "%s.profile" % (func.__name__) try: stats.add(fname) except IOError: pass stats.save(fname) # @yappi.profile(return_callback=aggregate) def sum(x, y): consumer_time() return x + y import yappi if __name__ == "__main__": # yappi.set_clock_type("wall") yappi.set_clock_type("cpu") yappi.start() for i in range(10): t = threading.Thread(target=sum, args=(100, i,), name="hello"+str(i)) t.start() main_thread = threading.currentThread() for t in threading.enumerate(): if t is not main_thread: t.join() yappi.get_func_stats().print_all(columns= {0:("name",50), 1:("ncall", 15), 2:("tsub", 18), 3: ("ttot", 18), 4:("tavg",18)}) yappi.get_thread_stats().print_all()