def teardown_profiler(self): if not self.is_profiling(): return path = os.environ["DRIVE_YAPPI"] if not os.path.exists(path): os.mkdir(path) report_path = os.path.join(path, self.id() + '-yappi-threads') with open(report_path, 'w') as fd: columns = { 0: ("name", 80), 1: ("tid", 15), 2: ("ttot", 8), 3: ("scnt", 10) } yappi.get_thread_stats().print_all(out=fd, columns=columns) report_path = os.path.join(path, self.id() + '-yappi-fcts') with open(report_path, 'w') as fd: columns = { 0: ("name", 80), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8) } stats = yappi.get_func_stats() stats.strip_dirs() stats.print_all(out=fd, columns=columns) log.debug("Profiler Report generated in '%s'", report_path)
def main(): profile_file = os.environ.get(config.PROFILE_ENV) if profile_file: import yappi yappi.start() try: print_version() options = parse_args() if options.version: exit(0) print("", file=sys.stderr) try: cfg = config.init_app_config(voyandz.app, options.conffile) except config.ConfigError: exit(1) server.autostart() voyandz.app.run(host=cfg['listenaddr'], port=cfg['listenport'], request_handler=logging.FormattedRequestHandler) finally: if profile_file: yappi.stop() pstats = yappi.convert2pstats(yappi.get_func_stats()) pstats.dump_stats(profile_file) with open(profile_file + ".threads", "w") as threads_file: yappi.get_thread_stats().print_all(threads_file)
def yappi_profile(path: Union[Callable[[], str], str] = None, wall_clock: bool = True): try: import yappi # pylint: disable=import-error except ImportError: print("Failed to run profiler, yappi is not installed") yield return yappi.set_clock_type("wall" if wall_clock else "cpu") yappi.start() yield yappi.stop() # pylint:disable=no-member if path: stats = yappi.get_func_stats() fpath = path() if callable(path) else path stats.save(fpath, "callgrind") else: yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() yappi.clear_stats()
def main(): profile_file = os.environ.get(PROFILE_ENV) if profile_file: # If profiling is enabled, start it. import yappi yappi.start() try: print_version() # Parse config. options = parse_args() if options.version: exit(0) print("", file=sys.stderr) if options.install: try: install() except InstallError as e: print("Install failed: {}".format(e), file=sys.stderr) exit(0) try: config.load(options.cfgfile) except config.ConfigError as cfg_error: print(cfg_error, file=sys.stderr) exit(1) # Run app. port = options.port or config.config.port adfotg.app.run(host='0.0.0.0', port=port) finally: if profile_file: # If profiling was enabled, stop it and generate reports. yappi.stop() pstats = yappi.convert2pstats(yappi.get_func_stats()) pstats.dump_stats(profile_file) with open(profile_file + ".threads", "w") as threads_file: yappi.get_thread_stats().print_all(threads_file)
def teardown_profiler(self): if not self.is_profiling(): return if not os.path.exists(YAPPI_PATH): os.mkdir(YAPPI_PATH) report_path = os.path.join( YAPPI_PATH, self.id() + '-' + sys.platform + '_yappi.txt') with open(report_path, 'w') as fd: fd.write('Threads\n=======\n') columns = { 0: ('name', 80), 1: ('tid', 15), 2: ('ttot', 8), 3: ('scnt', 10) } yappi.get_thread_stats().print_all(out=fd, columns=columns) fd.write('\n\n\nMethods\n=======\n') columns = { 0: ('name', 80), 1: ('ncall', 5), 2: ('tsub', 8), 3: ('ttot', 8), 4: ('tavg', 8) } stats = yappi.get_func_stats() stats.strip_dirs() stats.print_all(out=fd, columns=columns) log.debug('Profiler Report generated in %r', report_path)
def wrapper(*args, **kwargs): yappi.start() res = fn(*args, **kwargs) func_cols = {0: ("name", 160), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)} thread_cols = {0: ("name", 152), 1: ("id", 5), 2: ("tid", 16), 3: ("ttot", 8), 4: ("scnt", 8)} yappi.get_func_stats().sort("ttot").strip_dirs().print_all(columns=func_cols) yappi.get_thread_stats().print_all(columns=thread_cols) return res
def main(): print("Main TID: {}".format(gettid())) args = Bunch(channel=None, devices=[], generate=False, ui="main.ui") yappi.start() exit_value = epyqlib.__main__.main(args=args) yappi.stop() yappi.get_func_stats().save("yappi.stats", type="pstat") yappi.get_thread_stats().print_all() return exit_value
def main(self): yappi.start() # Pool connections to speed up our job # with Pool(processes=Threads().total()) as pool: # pool.map(self.run, range(30)) for i in range(100): self.run(i) yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all()
def cmd_stop(self, args): """stop profiling""" print "Profile results:" import yappi # We do the import here so that we won't barf if run normally and yappi not available yappi.get_func_stats().print_all( columns={0: ("name", 50), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)} ) yappi.get_thread_stats().print_all() yappi.stop()
def yappi_prof_call(func, *args): ''' https://code.google.com/p/yappi/wiki/usageyappi_v092 ''' import yappi yappi.start() result = func(*args) yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() return result
def ServerForever(server): if YappiProfile: try: yappi.start() server.serve_forever() except KeyboardInterrupt: print("Leaving") yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() else: server.serve_forever()
def main(): print('Main TID: {}'.format(gettid())) args = Bunch(channel=None, devices=[], generate=False, ui='main.ui') yappi.start() exit_value = epyqlib.__main__.main(args=args) yappi.stop() yappi.get_func_stats().save('yappi.stats', type='pstat') yappi.get_thread_stats().print_all() return exit_value
def main(): from optparse import OptionParser usage = "python -m greenlet_profiler [-b] [-s] [scriptfile] args ..." parser = OptionParser(usage=usage) parser.allow_interspersed_args = False parser.add_option( "-b", "--builtins", action="store_true", dest="profile_builtins", default=False, help="Profiles builtin functions when set. [default: False]") parser.add_option( "-s", "--single_thread", action="store_true", dest="profile_single_thread", default=False, help="Profiles only the thread that calls start(). [default: False]") clock_types = ['wall', 'cpu'] parser.add_option("-c", "--clock_type", dest="clock_type", type='choice', choices=clock_types, default='cpu', help="One of %s" % clock_types) options, args = parser.parse_args() if len(args) > 0: sys.path.insert(0, os.path.dirname(args[0])) set_clock_type(options.clock_type) start(options.profile_builtins, not options.profile_single_thread) if sys.version_info >= (3, 0): exec(compile(open(args[0]).read(), args[0], 'exec'), sys._getframe(1).f_globals, sys._getframe(1).f_locals) else: execfile(args[0], sys._getframe(1).f_globals, sys._getframe(1).f_locals) stop() get_func_stats().print_all() get_thread_stats().print_all() else: parser.print_usage() sys.exit(2)
def cmd_stop(self, args): '''stop profiling''' print "Profile results:" import yappi # We do the import here so that we won't barf if run normally and yappi not available yappi.get_func_stats().print_all( columns={ 0: ("name", 50), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8) }) yappi.get_thread_stats().print_all() yappi.stop()
def main(): try: if conf.ENABLE_PROFILING: yappi.start() launcher.main(sys.argv[1:]) yappi.stop() else: launcher.main(sys.argv[1:]) except KeyboardInterrupt: if conf.ENABLE_PROFILING: yappi.stop() print('Yappi result (func stats) ======================') yappi.get_func_stats().print_all() print('Yappi result (thread stats) ======================') yappi.get_thread_stats().print_all()
def main(): if not sys.argv[1:]: sys.argv.extend(["-h"]) args = parse_args(sys.argv[1:]) if has_benchmark and args.benchmark: if args.wall: yappi.set_clock_type("wall") yappi.start() if args.country != "list": mapper = Map( max_height=args.height, max_width=args.width, fill_char=args.fill, no_char=args.empty, outside_char=args.outside, blur=args.blur, method=args.method, surface=args.surface, negative=args.negative, is_unicode=args.unicode, ) render = mapper.Render(args.country) render.render_parallel() if args.stats: print() print_config(MapConfig()) print_config(PrinterConfig()) print_config(RenderConfig()) else: process_list_cmd() if has_benchmark and args.benchmark: yappi.get_func_stats().sort("ttot", "asc").print_all( columns={ 0: ("name", 120), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8), }) yappi.get_thread_stats().print_all()
def main(argv): parser = argparse.ArgumentParser(description='Convert a FASTA and QUAL pair of files into a FASQ file') parser.add_argument('fasta_filename', help='the FASTA format input file path') parser.add_argument('qual_filename', help='the QUAL format input file path') parser.add_argument('-q', action='store_true', help="run quietly") parser.add_argument('-d', action='store_true', help="print diagnostics to stderr") parser.add_argument('-z','--gzip', action='store_true', help='input & output are gzipped') parser.add_argument('-p', '--profile', action='store_true', help="profile this execution") parser.add_argument('-o', '--output', default=sys.stdout, type=argparse.FileType('w'), help='the output FASTQ file path') args = parser.parse_args() global debug_flag debug_flag = args.d debug_flag and sys.stderr.write("%r\n" % args) if args.profile: import yappi # https://code.google.com/p/yappi/wiki/apiyappi yappi.start(builtins=True) if args.gzip: import gzip fasta = gzip.open(args.fasta_filename) qual = gzip.open(args.qual_filename) outf = gzip.GzipFile(fileobj=args.output, mode='wb') else: fasta = open(args.fasta_filename) qual = open(args.qual_filename) outf = args.output # alphabet irrelevant (and makes it slow) records = PairedFastaQualIterator(fasta, qual, alphabet=IUPAC.ambiguous_dna) records = PairedFastaQualIterator(fasta, qual) count = SeqIO.write(records, outf, "fastq") outf.close() if args.profile: yappi.get_thread_stats().print_all(sys.stderr) yappi.get_func_stats().sort("subtime").print_all(sys.stderr) print("Converted %i records" % count)
def finish_yappi(): yappi.stop() print("[YAPPI WRITE]") stats = yappi.get_func_stats() # 'ystat' is Yappi internal format for stat_type in ["pstat", "callgrind"]: print("writing {}.{}".format(OUT_FILE, stat_type)) stats.save("{}.{}".format(OUT_FILE, stat_type), type=stat_type) print("\n[YAPPI FUNC_STATS]") print("writing {}.func_stats".format(OUT_FILE)) with open("{}.func_stats".format(OUT_FILE), "w") as fh: stats.print_all(out=fh) print("\n[YAPPI THREAD_STATS]") print("writing {}.thread_stats".format(OUT_FILE)) tstats = yappi.get_thread_stats() with open("{}.thread_stats".format(OUT_FILE), "w") as fh: tstats.print_all(out=fh) print("[YAPPI DONE]")
def finish_yappi(): print('[YAPPI STOP]') yappi.stop() print('[YAPPI WRITE]') stats = yappi.get_func_stats() for stat_type in ['pstat', 'callgrind', 'ystat']: print('writing run_stats.{}'.format(stat_type)) stats.save('run_stats.{}'.format(stat_type), type=stat_type) print('\n[YAPPI FUNC_STATS]') print('writing run_stats.func_stats') with open('run_stats.func_stats', 'w') as fh: stats.print_all(out=fh) print('\n[YAPPI THREAD_STATS]') print('writing run_stats.thread_stats') tstats = yappi.get_thread_stats() with open('run_stats.thread_stats', 'w') as fh: tstats.print_all(out=fh) print('[YAPPI OUT]')
def profile_stop(self): try: # noinspection PyUnresolvedReferences,PyPackageRequirements import yappi yappi.stop() columns = { 0: ("name", 36), 1: ("ncall", 7), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8) } yappi.get_func_stats().print_all(columns=columns) yappi.get_thread_stats().print_all() except Exception: sys.excepthook(*sys.exc_info())
def test_pause_resume(self): yappi.set_context_id_callback(lambda: self.context_id) yappi.set_clock_type('wall') # Start in context 0. self.context_id = 0 yappi.start() time.sleep(0.08) # Switch to context 1. self.context_id = 1 time.sleep(0.05) # Switch back to context 0. self.context_id = 0 time.sleep(0.07) yappi.stop() t_stats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(2, len(t_stats)) self.assertEqual(0, t_stats[0].id) self.assertEqual(2, t_stats[0].sched_count) self.assertTrue(0.15 < t_stats[0].ttot < 0.3) self.assertEqual(1, t_stats[1].id) self.assertEqual(1, t_stats[1].sched_count) # Context 1 was first scheduled 0.08 sec after context 0. self.assertTrue(0.1 < t_stats[1].ttot < 0.2 )
def finish_yappi(): yappi.stop() print('[YAPPI WRITE]') stats = yappi.get_func_stats() # 'ystat' is Yappi internal format for stat_type in ['pstat', 'callgrind']: print('writing {}.{}'.format(OUT_FILE, stat_type)) stats.save('{}.{}'.format(OUT_FILE, stat_type), type=stat_type) print('\n[YAPPI FUNC_STATS]') print('writing {}.func_stats'.format(OUT_FILE)) with open('{}.func_stats'.format(OUT_FILE), 'w') as fh: stats.print_all(out=fh) print('\n[YAPPI THREAD_STATS]') print('writing {}.thread_stats'.format(OUT_FILE)) tstats = yappi.get_thread_stats() with open('{}.thread_stats'.format(OUT_FILE), 'w') as fh: tstats.print_all(out=fh) print('[YAPPI DONE]')
def test_merge_multithreaded_stats(self): import threading import _yappi timings = {"a_1":2, "b_1":1} _yappi._set_test_timings(timings) def a(): pass def b(): pass yappi.start() t = threading.Thread(target=a) t.start() t.join() t = threading.Thread(target=b) t.start() t.join() yappi.get_func_stats().save("ystats1.ys") yappi.clear_stats() _yappi._set_test_timings(timings) self.assertEqual(len(yappi.get_func_stats()), 0) self.assertEqual(len(yappi.get_thread_stats()), 1) t = threading.Thread(target=a) t.start() t.join() self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0) self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1) yappi.get_func_stats().save("ystats2.ys") stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys",]) fsa = utils.find_stat_by_name(stats, "a") fsb = utils.find_stat_by_name(stats, "b") self.assertEqual(fsa.ncall, 2) self.assertEqual(fsb.ncall, 1) self.assertEqual(fsa.tsub, fsa.ttot, 4) self.assertEqual(fsb.tsub, fsb.ttot, 1)
def test_print_formatting(self): def a(): pass def b(): a() func_cols = {1: ("name", 48), 0: ("ncall", 5), 2: ("tsub", 8)} thread_cols = {1: ("name", 48), 0: ("ttot", 8)} yappi.start() a() b() yappi.stop() fs = yappi.get_func_stats() cs = fs[1].children ts = yappi.get_thread_stats() # fs.print_all(out=sys.stderr, columns={1:("name", 70), }) # cs.print_all(out=sys.stderr, columns=func_cols) # ts.print_all(out=sys.stderr, columns=thread_cols) # cs.print_all(out=sys.stderr, columns={}) self.assertRaises(yappi.YappiError, fs.print_all, columns={1: ("namee", 9)}) self.assertRaises(yappi.YappiError, cs.print_all, columns={1: ("dd", 0)}) self.assertRaises(yappi.YappiError, ts.print_all, columns={1: ("tidd", 0)})
def test_basic(self): import threading import time yappi.set_clock_type('wall') def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') self.assertTrue(tsa is not None) self.assertTrue(tsm is not None) # FIX: I see this fails sometimes?
def profile(): import yappi # yappi.set_clock_type("WALL") yappi.start() main() columns = { 0: ("name", 80), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8) } with open("yappi_prof.txt", "w") as of: yappi.get_func_stats().strip_dirs().sort("tsub").print_all( out=of, columns=columns) yappi.get_thread_stats().print_all()
def finish_yappi(): OUT_FILE = './tmp/pants' print('[YAPPI STOP]') yappi.stop() print('[YAPPI WRITE]') stats = yappi.get_func_stats() for stat_type in ['pstat', 'callgrind', 'ystat']: print('writing {}.{}'.format(OUT_FILE, stat_type)) stats.save('{}.{}'.format(OUT_FILE, stat_type), type=stat_type) print('\n[YAPPI FUNC_STATS]') print('writing {}.func_stats'.format(OUT_FILE)) with open('{}.func_stats'.format(OUT_FILE), 'w') as fh: stats.print_all(out=fh) print('\n[YAPPI THREAD_STATS]') print('writing {}.thread_stats'.format(OUT_FILE)) tstats = yappi.get_thread_stats() with open('{}.thread_stats'.format(OUT_FILE), 'w') as fh: tstats.print_all(out=fh) print('[YAPPI OUT]')
def test_pause_resume(self): yappi.set_context_id_callback(lambda: self.context_id) yappi.set_clock_type('wall') # Start in context 0. self.context_id = 0 yappi.start() time.sleep(0.08) # Switch to context 1. self.context_id = 1 time.sleep(0.05) # Switch back to context 0. self.context_id = 0 time.sleep(0.07) yappi.stop() t_stats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(2, len(t_stats)) self.assertEqual(0, t_stats[0].id) self.assertEqual(2, t_stats[0].sched_count) self.assertTrue(0.15 < t_stats[0].ttot < 0.7, t_stats[0].ttot) self.assertEqual(1, t_stats[1].id) self.assertEqual(1, t_stats[1].sched_count) # Context 1 was scheduled for 0.05 seconds during the run of the # profiler self.assert_almost_equal(t_stats[1].ttot, 0.05)
def test_callback(self): self.context_id = 0 yappi.set_context_id_callback(lambda: self.context_id) yappi.start() a() self.context_id = 1 a() self.context_id = 2 a() # Re-schedule context 1. self.context_id = 1 a() yappi.stop() threadstats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(3, len(threadstats)) self.assertEqual(0, threadstats[0].id) self.assertEqual(1, threadstats[1].id) self.assertEqual(2, threadstats[2].id) self.assertEqual(1, threadstats[0].sched_count) self.assertEqual(2, threadstats[1].sched_count) # Context 1 ran twice. self.assertEqual(1, threadstats[2].sched_count) funcstats = yappi.get_func_stats() self.assertEqual(4, utils.find_stat_by_name(funcstats, 'a').ncall)
def test_basic(self): yappi.set_clock_type('wall') def dummy(): pass def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') dummy() # call dummy to force ctx name to be retrieved again. self.assertTrue(tsa is not None) # TODO: I put dummy() to fix below, remove the comments after a while. self.assertTrue( # FIX: I see this fails sometimes? tsm is not None, 'Could not find "_MainThread". Found: %s' % (', '.join(utils.get_stat_names(tstats))))
def test_pause_resume(self): yappi.set_context_id_callback(lambda: self.context_id) yappi.set_clock_type('wall') # Start in context 0. self.context_id = 0 yappi.start() time.sleep(0.08) # Switch to context 1. self.context_id = 1 time.sleep(0.05) # Switch back to context 0. self.context_id = 0 time.sleep(0.07) yappi.stop() t_stats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(2, len(t_stats)) self.assertEqual(0, t_stats[0].id) self.assertEqual(2, t_stats[0].sched_count) self.assertAlmostEqual(0.2, t_stats[0].ttot, places=2) self.assertEqual(1, t_stats[1].id) self.assertEqual(1, t_stats[1].sched_count) # Context 1 was first scheduled 0.08 sec after context 0. self.assertAlmostEqual(0.12, t_stats[1].ttot, places=2)
def teardown_profiler(self): if not self.is_profiling(): return path = os.environ["DRIVE_YAPPI"] if not os.path.exists(path): os.mkdir(path) report_path = os.path.join(path, self.id() + '-yappi-threads') with open(report_path, 'w') as fd: columns = {0: ("name", 80), 1: ("tid", 15), 2: ("ttot", 8), 3: ("scnt", 10)} yappi.get_thread_stats().print_all(out=fd, columns=columns) report_path = os.path.join(path, self.id() + '-yappi-fcts') with open(report_path, 'w') as fd: columns = {0: ("name", 80), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)} stats = yappi.get_func_stats() stats.strip_dirs() stats.print_all(out=fd, columns=columns) log.debug("Profiler Report generated in '%s'", report_path)
def test_start_flags(self): self.assertEqual(_yappi._get_start_flags(), None) yappi.start() def a(): pass a() self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0) self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1) self.assertEqual(len(yappi.get_thread_stats()), 1)
def prof_threads(self): """ Return profile threads info """ import yappi i = yappi.get_thread_stats() out = six.StringIO() i.print_all(out=out) return out.getvalue()
def main(): try: if os.getenv('DEFAULT_SCORE_HOST') is not None: os.system("ssh-keyscan " + os.getenv('DEFAULT_SCORE_HOST') + " >> /root/.ssh/known_hosts") if conf.ENABLE_PROFILING: yappi.start() launcher.main(sys.argv[1:]) yappi.stop() else: launcher.main(sys.argv[1:]) except KeyboardInterrupt: if conf.ENABLE_PROFILING: yappi.stop() print('Yappi result (func stats) ======================') yappi.get_func_stats().print_all() print('Yappi result (thread stats) ======================') yappi.get_thread_stats().print_all()
async def get_prof_threads(): """ Get running threads info :return: """ import yappi i = yappi.get_thread_stats() out = StringIO() i.print_all(out=out) return out.getvalue()
async def print_profiler_stats(event_loop, time_delay): """ Print profiler statistics after a number of seconds. """ try: import yappi except ImportError: return await asyncio.sleep(time_delay, loop=event_loop) print("Printing profiler stats") yappi.get_func_stats().print_all(out=sys.stdout, columns={ 0: ("name", 60), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8) }) yappi.get_func_stats().save(PROFILING_OUTPUT_FILE, type="callgrind") yappi.get_thread_stats().print_all()
def test_subsequent_profile(self): import threading WORKER_COUNT = 5 def a(): pass def b(): pass def c(): pass _timings = {"a_1":3,"b_1":2,"c_1":1,} yappi.start() def g(): pass g() yappi.stop() yappi.clear_stats() _yappi._set_test_timings(_timings) yappi.start() _dummy = [] for i in range(WORKER_COUNT): t = threading.Thread(target=a) t.start() t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=b) t.start() _dummy.append(t) t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=a) t.start() t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=c) t.start() t.join() yappi.stop() yappi.start() def f(): pass f() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'a') fsb = utils.find_stat_by_name(stats, 'b') fsc = utils.find_stat_by_name(stats, 'c') self.assertEqual(fsa.ncall, 10) self.assertEqual(fsb.ncall, 5) self.assertEqual(fsc.ncall, 5) self.assertEqual(fsa.ttot, fsa.tsub, 30) self.assertEqual(fsb.ttot, fsb.tsub, 10) self.assertEqual(fsc.ttot, fsc.tsub, 5) # MACOSx optimizes by only creating one worker thread self.assertTrue(len(yappi.get_thread_stats()) >= 2)
def test_callback(self): self.context_id = 0 self.context_name = 'a' yappi.set_context_id_callback(lambda: self.context_id) yappi.set_context_name_callback(lambda: self.context_name) yappi.start() a() self.context_id = 1 self.context_name = 'b' a() # Re-schedule context 0. self.context_id = 0 self.context_name = 'a' a() yappi.stop() threadstats = yappi.get_thread_stats().sort('name', 'ascending') self.assertEqual(2, len(threadstats)) self.assertEqual(0, threadstats[0].id) self.assertEqual('a', threadstats[0].name) self.assertEqual(1, threadstats[1].id) self.assertEqual('b', threadstats[1].name)
if __name__ == "__main__": if DEBUG: sys.argv.append("-h") if TESTRUN: import doctest doctest.testmod() if PROFILE: import pstats #Use yappi in multithreaded context import yappi yappi.start() exit_code = main() yappi.stop() stats = yappi.get_func_stats() tstats = yappi.get_thread_stats() STAT_FILE = 'profile_stats{}' stats.save(STAT_FILE.format('.pstat'), type='pstat') with open(STAT_FILE.format('.txt'), 'w') as fh : # pstat format better than yappi format ps = pstats.Stats(STAT_FILE.format('.pstat'), stream = fh) ps.sort_stats('time') ps.print_stats() ps.sort_stats('cumulative') ps.print_stats() #append thread stats tstats.print_all(out = fh) #add function stat with yappi format stats.print_all(out=fh)
def lineReceived(self, line): anon_tunnel = self.anon_tunnel profile = self.profile if line == 'threads': for thread in threading.enumerate(): print "%s \t %d" % (thread.name, thread.ident) elif line == 'p': if profile: for func_stats in yappi.get_func_stats().sort("subtime")[:50]: print "YAPPI: %10dx %10.3fs" % (func_stats.ncall, func_stats.tsub), func_stats.name else: logger.error("Profiling disabled!") elif line == 'P': if profile: filename = 'callgrindc_%d.yappi' % anon_tunnel.dispersy.lan_address[1] yappi.get_func_stats().save(filename, type='callgrind') else: logger.error("Profiling disabled!") elif line == 't': if profile: yappi.get_thread_stats().sort("totaltime").print_all() else: logger.error("Profiling disabled!") elif line == 'c': print "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype" for circuit_id, circuit in anon_tunnel.community.circuits.items(): info_hash = circuit.info_hash.encode('hex')[:10] if circuit.info_hash else '?' print "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % (circuit_id, circuit.first_hop[0], circuit.first_hop[1], circuit.goal_hops, len(circuit.hops), circuit.bytes_down / 1024.0 / 1024.0, circuit.bytes_up / 1024.0 / 1024.0, info_hash, circuit.ctype) elif line.startswith('s'): cur_path = os.getcwd() line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] if not os.path.exists(filename): logger.info("Creating torrent..") with open(filename, 'wb') as fp: fp.write(os.urandom(50 * 1024 * 1024)) tdef = TorrentDef() tdef.add_content(os.path.join(cur_path, filename)) tdef.set_tracker("udp://fake.net/announce") tdef.set_private() tdef.finalize() tdef.save(os.path.join(cur_path, filename + '.torrent')) else: logger.info("Loading existing torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10])) defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(cur_path) anon_tunnel.session.lm.threadpool.call(0, anon_tunnel.session.start_download, tdef, dscfg) elif line.startswith('i'): # Introduce dispersy port from other main peer to this peer line_split = line.split(' ') to_introduce_ip = line_split[1] to_introduce_port = int(line_split[2]) self.anon_tunnel.community.add_discovered_candidate(Candidate((to_introduce_ip, to_introduce_port), tunnel=False)) elif line.startswith('d'): line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] logger.info("Loading torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("Loading torrent done") defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(os.path.join(os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port())) def start_download(): def cb(ds): logger.info('Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' % (tdef.get_infohash().encode('hex')[:10], ds.get_current_speed('down'), ds.get_progress(), dlstatus_strings[ds.get_status()], sum(ds.get_num_seeds_peers()), sum(1 for _ in anon_tunnel.community.dispersy_yield_verified_candidates()))) return 1.0, False download = anon_tunnel.session.start_download(tdef, dscfg) download.set_state_callback(cb, delay=1) anon_tunnel.session.lm.threadpool.call(0, start_download) elif line == 'q': anon_tunnel.stop() return elif line == 'r': print "circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)" from_to = anon_tunnel.community.relay_from_to for key in from_to.keys(): relay = from_to[key] logger.info("%s-->\t%s\t\t%.2f" % ((key[0], key[1]), (relay.sock_addr, relay.circuit_id), relay.bytes[1] / 1024.0 / 1024.0,))
import yappi #yappi.set_clock_type("waLL") #yappi.start(builtins=True) yappi.start() class A: def bar(self): pass def foo(): def inner_foo(): pass import time time.sleep(2.0) for i in range(20000000): pass a = A() a.bar() inner_foo() foo() #yappi.write_callgrind_stats() yappi.get_func_stats().sort("totaLTiMe").print_all() yappi.get_thread_stats().print_all() #import cProfile #cProfile.run('foo()', 'fooprof') #import pstats #p = pstats.Stats('fooprof') #p.strip_dirs().sort_stats(-1).print_stats()
def run_and_get_thread_stats(func, *args, **kwargs): run_with_yappi(func, *args, **kwargs) return yappi.get_thread_stats()
def test_ctx_stats(self): from threading import Thread DUMMY_WORKER_COUNT = 5 yappi.start() class DummyThread(Thread): pass def dummy(): pass def dummy_worker(): pass for i in range(DUMMY_WORKER_COUNT): t = DummyThread(target=dummy_worker) t.start() t.join() yappi.stop() stats = yappi.get_thread_stats() tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is not None) yappi.clear_stats() time.sleep(1.0) _timings = {"a_1": 6, "b_1": 5, "c_1": 3, "d_1": 1, "a_2": 4, "b_2": 3, "c_2": 2, "d_2": 1} _yappi._set_test_timings(_timings) class Thread1(Thread): pass class Thread2(Thread): pass def a(): b() def b(): c() def c(): d() def d(): time.sleep(0.6) yappi.set_clock_type("wall") yappi.start() t1 = Thread1(target=a) t1.start() t2 = Thread2(target=a) t2.start() t1.join() t2.join() stats = yappi.get_thread_stats() # the fist clear_stats clears the context table? tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is None) tst1 = utils.find_stat_by_name(stats, "Thread1") tst2 = utils.find_stat_by_name(stats, "Thread2") tsmain = utils.find_stat_by_name(stats, "_MainThread") dummy() # call dummy to force ctx name to be retrieved again. self.assertTrue(len(stats) == 3) self.assertTrue(tst1 is not None) self.assertTrue(tst2 is not None) # TODO: I put dummy() to fix below, remove the comments after a while. self.assertTrue( # FIX: I see this fails sometimes tsmain is not None, 'Could not find "_MainThread". Found: %s' % (", ".join(utils.get_stat_names(stats))) ) self.assertTrue(1.0 > tst2.ttot >= 0.5) self.assertTrue(1.0 > tst1.ttot >= 0.5) # test sorting of the ctx stats stats = stats.sort("totaltime", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot >= stat.ttot) prev_stat = stat stats = stats.sort("totaltime", "asc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot <= stat.ttot) prev_stat = stat stats = stats.sort("schedcount", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.sched_count >= stat.sched_count) prev_stat = stat stats = stats.sort("name", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.name >= stat.name) prev_stat = stat self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sorttype_arg") self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sortorder_arg")
def test_ctx_stats(self): from threading import Thread DUMMY_WORKER_COUNT = 5 yappi.start() class DummyThread(Thread): pass def dummy_worker(): pass for i in range(DUMMY_WORKER_COUNT): t = DummyThread(target=dummy_worker) t.start() t.join() yappi.stop() stats = yappi.get_thread_stats() tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is not None) yappi.clear_stats() import time time.sleep(1.0) _timings = {"a_1":6,"b_1":5,"c_1":3, "d_1":1, "a_2":4,"b_2":3,"c_2":2, "d_2":1} _yappi._set_test_timings(_timings) class Thread1(Thread): pass class Thread2(Thread): pass def a(): b() def b(): c() def c(): d() def d(): time.sleep(0.6) yappi.set_clock_type("wall") yappi.start() t1 = Thread1(target=a) t1.start() t2 = Thread2(target=a) t2.start() t1.join() t2.join() stats = yappi.get_thread_stats() # the fist clear_stats clears the context table? tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is None) tst1 = utils.find_stat_by_name(stats, "Thread1") tst2 = utils.find_stat_by_name(stats, "Thread2") tsmain = utils.find_stat_by_name(stats, "_MainThread") #stats.print_all() self.assertTrue(len(stats) == 3) self.assertTrue(tst1 is not None) self.assertTrue(tst2 is not None) self.assertTrue(tsmain is not None) # I see this fails sometimes, probably # because Py_ImportNoBlock() fails to import and get the thread class name # sometimes. self.assertTrue(1.0 > tst2.ttot >= 0.5) self.assertTrue(1.0 > tst1.ttot >= 0.5) # test sorting of the ctx stats stats = stats.sort("totaltime", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot >= stat.ttot) prev_stat = stat stats = stats.sort("totaltime", "asc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot <= stat.ttot) prev_stat = stat stats = stats.sort("schedcount", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.sched_count >= stat.sched_count) prev_stat = stat stats = stats.sort("name", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.name >= stat.name) prev_stat = stat self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sorttype_arg") self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sortorder_arg")
import yappi def test_function(): pass class T(Thread): def __init__(self): Thread.__init__(self) def run(self): # takes about 5 seconds for i in xrange(100): self.test_method() test_function() def test_method(self): sleep(random.random() / 10) yappi.start() ####################### threads = [T() for i in xrange(3)] for t in threads: t.start() for i in xrange(100): test_function() for t in threads: t.join() ####################### yappi.stop() yappi.get_func_stats().sort("totaltime").print_all() yappi.get_thread_stats().sort("totaltime").print_all()
except KeyboardInterrupt: if mpstate.settings.requireexit: print("Interrupt caught. Use 'exit' to quit MAVProxy.") #Just lost the map and console, get them back: for (m,pm) in mpstate.modules: if m.name in ["map", "console"]: if hasattr(m, 'unload'): try: m.unload() except Exception: pass reload(m) m.init(mpstate) else: mpstate.status.exit = True sys.exit(1) if opts.profile: yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() #this loop executes after leaving the above loop and is for cleanup on exit for (m,pm) in mpstate.modules: if hasattr(m, 'unload'): print("Unloading module %s" % m.name) m.unload() sys.exit(1)
def sys_exit_call(signum, frame): print "profiling result:" yappi.get_func_stats().print_all(columns={0:("name",100), 1:("ncall", 15), 2:("tsub", 8), 3:("ttot", 8), 4:("tavg",8)}) yappi.get_thread_stats().print_all() sys.exit(1)
print("\nConnection successful!\n") except Exception as err: print("\nProblem connecting {0}\n ".format (err)) pass sys.exit(0) elif (cmdline_arg_isset("-yappi")): print("Running with yappi profiler") import yappi yappi.start() SAMPListener() profiler = open("profile.data", "w") yappi.get_func_stats().debug_print() #print_all() yappi.get_thread_stats().print_all(out=profiler) profiler.close() else: print("Starting logger") options = read_options_from_commandline(None, ignore_errors=True) options = podi_logging.setup_logging(options) SAMPListener() print("Shutting down QuickReduce logging") podi_logging.shutdown_logging(options)
output_error("Specify scene file", output_function) exit(1) else: scene_desc = pickle.load(sys.stdin) pstd = PSTD(args.multi_threaded, args.write_plot, args.write_array, scene_desc, output_function) out = sys.__stdout__ if args.file: print("Writing to %s"%args.file) out = open(args.file,'w') out.write("openPSTD --- %s\n"%datetime.now().strftime('%d-%m-%y %H:%M')) out.write("Profile output\n" "Scene file %s\n" "Parallel: %s\n" "Plotting: %s\n" "Writing arrays: %s\n\n"%(args.scene_file,args.multi_threaded,args.write_plot,args.write_array)) time_start = time.time() yappi.start() pstd.run() yappi.stop() time_passed = time.time() - time_start out.write("Time passed: %.3f\n"%time_passed) funcstats = yappi.get_func_stats() funcstats.sort(sort_type='tsub', sort_order="desc") funcstats.print_all(out=out,columns={0:("name",36), 1:("ncall", 8), 2:("tsub", 8), 3:("ttot", 8), 4:("tavg",8)}) yappi.get_thread_stats().print_all(out=out) if args.file: out.close() exit() # yappi.start() # foo()