def teardown_profiler(self):
if not self.is_profiling():
return
path = os.environ["DRIVE_YAPPI"]
if not os.path.exists(path):
os.mkdir(path)
report_path = os.path.join(path, self.id() + '-yappi-threads')
with open(report_path, 'w') as fd:
columns = {
0: ("name", 80),
1: ("tid", 15),
2: ("ttot", 8),
3: ("scnt", 10)
}
yappi.get_thread_stats().print_all(out=fd, columns=columns)
report_path = os.path.join(path, self.id() + '-yappi-fcts')
with open(report_path, 'w') as fd:
columns = {
0: ("name", 80),
1: ("ncall", 5),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8)
}
stats = yappi.get_func_stats()
stats.strip_dirs()
stats.print_all(out=fd, columns=columns)
log.debug("Profiler Report generated in '%s'", report_path)
def main():
profile_file = os.environ.get(config.PROFILE_ENV)
if profile_file:
import yappi
yappi.start()
try:
print_version()
options = parse_args()
if options.version:
exit(0)
print("", file=sys.stderr)
try:
cfg = config.init_app_config(voyandz.app, options.conffile)
except config.ConfigError:
exit(1)
server.autostart()
voyandz.app.run(host=cfg['listenaddr'], port=cfg['listenport'],
request_handler=logging.FormattedRequestHandler)
finally:
if profile_file:
yappi.stop()
pstats = yappi.convert2pstats(yappi.get_func_stats())
pstats.dump_stats(profile_file)
with open(profile_file + ".threads", "w") as threads_file:
yappi.get_thread_stats().print_all(threads_file)
def yappi_profile(path: Union[Callable[[], str], str] = None,
wall_clock: bool = True):
try:
import yappi # pylint: disable=import-error
except ImportError:
print("Failed to run profiler, yappi is not installed")
yield
return
yappi.set_clock_type("wall" if wall_clock else "cpu")
yappi.start()
yield
yappi.stop()
# pylint:disable=no-member
if path:
stats = yappi.get_func_stats()
fpath = path() if callable(path) else path
stats.save(fpath, "callgrind")
else:
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
yappi.clear_stats()
def main():
profile_file = os.environ.get(PROFILE_ENV)
if profile_file:
# If profiling is enabled, start it.
import yappi
yappi.start()
try:
print_version()
# Parse config.
options = parse_args()
if options.version:
exit(0)
print("", file=sys.stderr)
if options.install:
try:
install()
except InstallError as e:
print("Install failed: {}".format(e), file=sys.stderr)
exit(0)
try:
config.load(options.cfgfile)
except config.ConfigError as cfg_error:
print(cfg_error, file=sys.stderr)
exit(1)
# Run app.
port = options.port or config.config.port
adfotg.app.run(host='0.0.0.0', port=port)
finally:
if profile_file:
# If profiling was enabled, stop it and generate reports.
yappi.stop()
pstats = yappi.convert2pstats(yappi.get_func_stats())
pstats.dump_stats(profile_file)
with open(profile_file + ".threads", "w") as threads_file:
yappi.get_thread_stats().print_all(threads_file)
def teardown_profiler(self):
if not self.is_profiling():
return
if not os.path.exists(YAPPI_PATH):
os.mkdir(YAPPI_PATH)
report_path = os.path.join(
YAPPI_PATH,
self.id() + '-' + sys.platform + '_yappi.txt')
with open(report_path, 'w') as fd:
fd.write('Threads\n=======\n')
columns = {
0: ('name', 80),
1: ('tid', 15),
2: ('ttot', 8),
3: ('scnt', 10)
}
yappi.get_thread_stats().print_all(out=fd, columns=columns)
fd.write('\n\n\nMethods\n=======\n')
columns = {
0: ('name', 80),
1: ('ncall', 5),
2: ('tsub', 8),
3: ('ttot', 8),
4: ('tavg', 8)
}
stats = yappi.get_func_stats()
stats.strip_dirs()
stats.print_all(out=fd, columns=columns)
log.debug('Profiler Report generated in %r', report_path)
def wrapper(*args, **kwargs):
yappi.start()
res = fn(*args, **kwargs)
func_cols = {0: ("name", 160), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)}
thread_cols = {0: ("name", 152), 1: ("id", 5), 2: ("tid", 16), 3: ("ttot", 8), 4: ("scnt", 8)}
yappi.get_func_stats().sort("ttot").strip_dirs().print_all(columns=func_cols)
yappi.get_thread_stats().print_all(columns=thread_cols)
return res
def main():
print("Main TID: {}".format(gettid()))
args = Bunch(channel=None, devices=[], generate=False, ui="main.ui")
yappi.start()
exit_value = epyqlib.__main__.main(args=args)
yappi.stop()
yappi.get_func_stats().save("yappi.stats", type="pstat")
yappi.get_thread_stats().print_all()
return exit_value
def main(self):
yappi.start()
# Pool connections to speed up our job
# with Pool(processes=Threads().total()) as pool:
# pool.map(self.run, range(30))
for i in range(100):
self.run(i)
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
def cmd_stop(self, args):
"""stop profiling"""
print "Profile results:"
import yappi # We do the import here so that we won't barf if run normally and yappi not available
yappi.get_func_stats().print_all(
columns={0: ("name", 50), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)}
)
yappi.get_thread_stats().print_all()
yappi.stop()
def yappi_prof_call(func, *args):
'''
https://code.google.com/p/yappi/wiki/usageyappi_v092
'''
import yappi
yappi.start()
result = func(*args)
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
return result
def yappi_prof_call(func, *args):
'''
https://code.google.com/p/yappi/wiki/usageyappi_v092
'''
import yappi
yappi.start()
result = func(*args)
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
return result
def ServerForever(server):
if YappiProfile:
try:
yappi.start()
server.serve_forever()
except KeyboardInterrupt:
print("Leaving")
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
else:
server.serve_forever()
def main():
print('Main TID: {}'.format(gettid()))
args = Bunch(channel=None,
devices=[],
generate=False,
ui='main.ui')
yappi.start()
exit_value = epyqlib.__main__.main(args=args)
yappi.stop()
yappi.get_func_stats().save('yappi.stats', type='pstat')
yappi.get_thread_stats().print_all()
return exit_value
def main():
from optparse import OptionParser
usage = "python -m greenlet_profiler [-b] [-s] [scriptfile] args ..."
parser = OptionParser(usage=usage)
parser.allow_interspersed_args = False
parser.add_option(
"-b",
"--builtins",
action="store_true",
dest="profile_builtins",
default=False,
help="Profiles builtin functions when set. [default: False]")
parser.add_option(
"-s",
"--single_thread",
action="store_true",
dest="profile_single_thread",
default=False,
help="Profiles only the thread that calls start(). [default: False]")
clock_types = ['wall', 'cpu']
parser.add_option("-c",
"--clock_type",
dest="clock_type",
type='choice',
choices=clock_types,
default='cpu',
help="One of %s" % clock_types)
options, args = parser.parse_args()
if len(args) > 0:
sys.path.insert(0, os.path.dirname(args[0]))
set_clock_type(options.clock_type)
start(options.profile_builtins, not options.profile_single_thread)
if sys.version_info >= (3, 0):
exec(compile(open(args[0]).read(), args[0], 'exec'),
sys._getframe(1).f_globals,
sys._getframe(1).f_locals)
else:
execfile(args[0],
sys._getframe(1).f_globals,
sys._getframe(1).f_locals)
stop()
get_func_stats().print_all()
get_thread_stats().print_all()
else:
parser.print_usage()
sys.exit(2)
def cmd_stop(self, args):
'''stop profiling'''
print "Profile results:"
import yappi # We do the import here so that we won't barf if run normally and yappi not available
yappi.get_func_stats().print_all(
columns={
0: ("name", 50),
1: ("ncall", 5),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8)
})
yappi.get_thread_stats().print_all()
yappi.stop()
def main():
try:
if conf.ENABLE_PROFILING:
yappi.start()
launcher.main(sys.argv[1:])
yappi.stop()
else:
launcher.main(sys.argv[1:])
except KeyboardInterrupt:
if conf.ENABLE_PROFILING:
yappi.stop()
print('Yappi result (func stats) ======================')
yappi.get_func_stats().print_all()
print('Yappi result (thread stats) ======================')
yappi.get_thread_stats().print_all()
def main():
if not sys.argv[1:]:
sys.argv.extend(["-h"])
args = parse_args(sys.argv[1:])
if has_benchmark and args.benchmark:
if args.wall:
yappi.set_clock_type("wall")
yappi.start()
if args.country != "list":
mapper = Map(
max_height=args.height,
max_width=args.width,
fill_char=args.fill,
no_char=args.empty,
outside_char=args.outside,
blur=args.blur,
method=args.method,
surface=args.surface,
negative=args.negative,
is_unicode=args.unicode,
)
render = mapper.Render(args.country)
render.render_parallel()
if args.stats:
print()
print_config(MapConfig())
print_config(PrinterConfig())
print_config(RenderConfig())
else:
process_list_cmd()
if has_benchmark and args.benchmark:
yappi.get_func_stats().sort("ttot", "asc").print_all(
columns={
0: ("name", 120),
1: ("ncall", 5),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8),
})
yappi.get_thread_stats().print_all()
def main(argv):
parser = argparse.ArgumentParser(description='Convert a FASTA and QUAL pair of files into a FASQ file')
parser.add_argument('fasta_filename',
help='the FASTA format input file path')
parser.add_argument('qual_filename',
help='the QUAL format input file path')
parser.add_argument('-q', action='store_true',
help="run quietly")
parser.add_argument('-d', action='store_true',
help="print diagnostics to stderr")
parser.add_argument('-z','--gzip', action='store_true',
help='input & output are gzipped')
parser.add_argument('-p', '--profile', action='store_true',
help="profile this execution")
parser.add_argument('-o', '--output',
default=sys.stdout, type=argparse.FileType('w'),
help='the output FASTQ file path')
args = parser.parse_args()
global debug_flag
debug_flag = args.d
debug_flag and sys.stderr.write("%r\n" % args)
if args.profile:
import yappi # https://code.google.com/p/yappi/wiki/apiyappi
yappi.start(builtins=True)
if args.gzip:
import gzip
fasta = gzip.open(args.fasta_filename)
qual = gzip.open(args.qual_filename)
outf = gzip.GzipFile(fileobj=args.output, mode='wb')
else:
fasta = open(args.fasta_filename)
qual = open(args.qual_filename)
outf = args.output
# alphabet irrelevant (and makes it slow) records = PairedFastaQualIterator(fasta, qual, alphabet=IUPAC.ambiguous_dna)
records = PairedFastaQualIterator(fasta, qual)
count = SeqIO.write(records, outf, "fastq")
outf.close()
if args.profile:
yappi.get_thread_stats().print_all(sys.stderr)
yappi.get_func_stats().sort("subtime").print_all(sys.stderr)
print("Converted %i records" % count)
def finish_yappi():
yappi.stop()
print("[YAPPI WRITE]")
stats = yappi.get_func_stats()
# 'ystat' is Yappi internal format
for stat_type in ["pstat", "callgrind"]:
print("writing {}.{}".format(OUT_FILE, stat_type))
stats.save("{}.{}".format(OUT_FILE, stat_type), type=stat_type)
print("\n[YAPPI FUNC_STATS]")
print("writing {}.func_stats".format(OUT_FILE))
with open("{}.func_stats".format(OUT_FILE), "w") as fh:
stats.print_all(out=fh)
print("\n[YAPPI THREAD_STATS]")
print("writing {}.thread_stats".format(OUT_FILE))
tstats = yappi.get_thread_stats()
with open("{}.thread_stats".format(OUT_FILE), "w") as fh:
tstats.print_all(out=fh)
print("[YAPPI DONE]")
def finish_yappi():
print('[YAPPI STOP]')
yappi.stop()
print('[YAPPI WRITE]')
stats = yappi.get_func_stats()
for stat_type in ['pstat', 'callgrind', 'ystat']:
print('writing run_stats.{}'.format(stat_type))
stats.save('run_stats.{}'.format(stat_type), type=stat_type)
print('\n[YAPPI FUNC_STATS]')
print('writing run_stats.func_stats')
with open('run_stats.func_stats', 'w') as fh:
stats.print_all(out=fh)
print('\n[YAPPI THREAD_STATS]')
print('writing run_stats.thread_stats')
tstats = yappi.get_thread_stats()
with open('run_stats.thread_stats', 'w') as fh:
tstats.print_all(out=fh)
print('[YAPPI OUT]')
def profile_stop(self):
try:
# noinspection PyUnresolvedReferences,PyPackageRequirements
import yappi
yappi.stop()
columns = {
0: ("name", 36),
1: ("ncall", 7),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8)
}
yappi.get_func_stats().print_all(columns=columns)
yappi.get_thread_stats().print_all()
except Exception:
sys.excepthook(*sys.exc_info())
def test_pause_resume(self):
yappi.set_context_id_callback(lambda: self.context_id)
yappi.set_clock_type('wall')
# Start in context 0.
self.context_id = 0
yappi.start()
time.sleep(0.08)
# Switch to context 1.
self.context_id = 1
time.sleep(0.05)
# Switch back to context 0.
self.context_id = 0
time.sleep(0.07)
yappi.stop()
t_stats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(2, len(t_stats))
self.assertEqual(0, t_stats[0].id)
self.assertEqual(2, t_stats[0].sched_count)
self.assertTrue(0.15 < t_stats[0].ttot < 0.3)
self.assertEqual(1, t_stats[1].id)
self.assertEqual(1, t_stats[1].sched_count)
# Context 1 was first scheduled 0.08 sec after context 0.
self.assertTrue(0.1 < t_stats[1].ttot < 0.2 )
def finish_yappi():
yappi.stop()
print('[YAPPI WRITE]')
stats = yappi.get_func_stats()
# 'ystat' is Yappi internal format
for stat_type in ['pstat', 'callgrind']:
print('writing {}.{}'.format(OUT_FILE, stat_type))
stats.save('{}.{}'.format(OUT_FILE, stat_type), type=stat_type)
print('\n[YAPPI FUNC_STATS]')
print('writing {}.func_stats'.format(OUT_FILE))
with open('{}.func_stats'.format(OUT_FILE), 'w') as fh:
stats.print_all(out=fh)
print('\n[YAPPI THREAD_STATS]')
print('writing {}.thread_stats'.format(OUT_FILE))
tstats = yappi.get_thread_stats()
with open('{}.thread_stats'.format(OUT_FILE), 'w') as fh:
tstats.print_all(out=fh)
print('[YAPPI DONE]')
def test_merge_multithreaded_stats(self):
import threading
import _yappi
timings = {"a_1":2, "b_1":1}
_yappi._set_test_timings(timings)
def a(): pass
def b(): pass
yappi.start()
t = threading.Thread(target=a)
t.start()
t.join()
t = threading.Thread(target=b)
t.start()
t.join()
yappi.get_func_stats().save("ystats1.ys")
yappi.clear_stats()
_yappi._set_test_timings(timings)
self.assertEqual(len(yappi.get_func_stats()), 0)
self.assertEqual(len(yappi.get_thread_stats()), 1)
t = threading.Thread(target=a)
t.start()
t.join()
self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0)
self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1)
yappi.get_func_stats().save("ystats2.ys")
stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys",])
fsa = utils.find_stat_by_name(stats, "a")
fsb = utils.find_stat_by_name(stats, "b")
self.assertEqual(fsa.ncall, 2)
self.assertEqual(fsb.ncall, 1)
self.assertEqual(fsa.tsub, fsa.ttot, 4)
self.assertEqual(fsb.tsub, fsb.ttot, 1)
def test_print_formatting(self):
def a():
pass
def b():
a()
func_cols = {1: ("name", 48), 0: ("ncall", 5), 2: ("tsub", 8)}
thread_cols = {1: ("name", 48), 0: ("ttot", 8)}
yappi.start()
a()
b()
yappi.stop()
fs = yappi.get_func_stats()
cs = fs[1].children
ts = yappi.get_thread_stats()
# fs.print_all(out=sys.stderr, columns={1:("name", 70), })
# cs.print_all(out=sys.stderr, columns=func_cols)
# ts.print_all(out=sys.stderr, columns=thread_cols)
# cs.print_all(out=sys.stderr, columns={})
self.assertRaises(yappi.YappiError, fs.print_all, columns={1: ("namee", 9)})
self.assertRaises(yappi.YappiError, cs.print_all, columns={1: ("dd", 0)})
self.assertRaises(yappi.YappiError, ts.print_all, columns={1: ("tidd", 0)})
def test_basic(self):
import threading
import time
yappi.set_clock_type('wall')
def a():
time.sleep(0.2)
class Worker1(threading.Thread):
def a(self):
time.sleep(0.3)
def run(self):
self.a()
yappi.start(builtins=False, profile_threads=True)
c = Worker1()
c.start()
c.join()
a()
stats = yappi.get_func_stats()
fsa1 = utils.find_stat_by_name(stats, 'Worker1.a')
fsa2 = utils.find_stat_by_name(stats, 'a')
self.assertTrue(fsa1 is not None)
self.assertTrue(fsa2 is not None)
self.assertTrue(fsa1.ttot > 0.2)
self.assertTrue(fsa2.ttot > 0.1)
tstats = yappi.get_thread_stats()
self.assertEqual(len(tstats), 2)
tsa = utils.find_stat_by_name(tstats, 'Worker1')
tsm = utils.find_stat_by_name(tstats, '_MainThread')
self.assertTrue(tsa is not None)
self.assertTrue(tsm is not None) # FIX: I see this fails sometimes?
def profile():
import yappi
# yappi.set_clock_type("WALL")
yappi.start()
main()
columns = {
0: ("name", 80),
1: ("ncall", 5),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8)
}
with open("yappi_prof.txt", "w") as of:
yappi.get_func_stats().strip_dirs().sort("tsub").print_all(
out=of, columns=columns)
yappi.get_thread_stats().print_all()
def finish_yappi():
OUT_FILE = './tmp/pants'
print('[YAPPI STOP]')
yappi.stop()
print('[YAPPI WRITE]')
stats = yappi.get_func_stats()
for stat_type in ['pstat', 'callgrind', 'ystat']:
print('writing {}.{}'.format(OUT_FILE, stat_type))
stats.save('{}.{}'.format(OUT_FILE, stat_type), type=stat_type)
print('\n[YAPPI FUNC_STATS]')
print('writing {}.func_stats'.format(OUT_FILE))
with open('{}.func_stats'.format(OUT_FILE), 'w') as fh:
stats.print_all(out=fh)
print('\n[YAPPI THREAD_STATS]')
print('writing {}.thread_stats'.format(OUT_FILE))
tstats = yappi.get_thread_stats()
with open('{}.thread_stats'.format(OUT_FILE), 'w') as fh:
tstats.print_all(out=fh)
print('[YAPPI OUT]')
def test_pause_resume(self):
yappi.set_context_id_callback(lambda: self.context_id)
yappi.set_clock_type('wall')
# Start in context 0.
self.context_id = 0
yappi.start()
time.sleep(0.08)
# Switch to context 1.
self.context_id = 1
time.sleep(0.05)
# Switch back to context 0.
self.context_id = 0
time.sleep(0.07)
yappi.stop()
t_stats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(2, len(t_stats))
self.assertEqual(0, t_stats[0].id)
self.assertEqual(2, t_stats[0].sched_count)
self.assertTrue(0.15 < t_stats[0].ttot < 0.7, t_stats[0].ttot)
self.assertEqual(1, t_stats[1].id)
self.assertEqual(1, t_stats[1].sched_count)
# Context 1 was scheduled for 0.05 seconds during the run of the
# profiler
self.assert_almost_equal(t_stats[1].ttot, 0.05)
def test_callback(self):
self.context_id = 0
yappi.set_context_id_callback(lambda: self.context_id)
yappi.start()
a()
self.context_id = 1
a()
self.context_id = 2
a()
# Re-schedule context 1.
self.context_id = 1
a()
yappi.stop()
threadstats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(3, len(threadstats))
self.assertEqual(0, threadstats[0].id)
self.assertEqual(1, threadstats[1].id)
self.assertEqual(2, threadstats[2].id)
self.assertEqual(1, threadstats[0].sched_count)
self.assertEqual(2, threadstats[1].sched_count) # Context 1 ran twice.
self.assertEqual(1, threadstats[2].sched_count)
funcstats = yappi.get_func_stats()
self.assertEqual(4, utils.find_stat_by_name(funcstats, 'a').ncall)
def test_basic(self):
yappi.set_clock_type('wall')
def dummy():
pass
def a():
time.sleep(0.2)
class Worker1(threading.Thread):
def a(self):
time.sleep(0.3)
def run(self):
self.a()
yappi.start(builtins=False, profile_threads=True)
c = Worker1()
c.start()
c.join()
a()
stats = yappi.get_func_stats()
fsa1 = utils.find_stat_by_name(stats, 'Worker1.a')
fsa2 = utils.find_stat_by_name(stats, 'a')
self.assertTrue(fsa1 is not None)
self.assertTrue(fsa2 is not None)
self.assertTrue(fsa1.ttot > 0.2)
self.assertTrue(fsa2.ttot > 0.1)
tstats = yappi.get_thread_stats()
self.assertEqual(len(tstats), 2)
tsa = utils.find_stat_by_name(tstats, 'Worker1')
tsm = utils.find_stat_by_name(tstats, '_MainThread')
dummy() # call dummy to force ctx name to be retrieved again.
self.assertTrue(tsa is not None)
# TODO: I put dummy() to fix below, remove the comments after a while.
self.assertTrue( # FIX: I see this fails sometimes?
tsm is not None,
'Could not find "_MainThread". Found: %s' % (', '.join(utils.get_stat_names(tstats))))
def test_callback(self):
self.context_id = 0
yappi.set_context_id_callback(lambda: self.context_id)
yappi.start()
a()
self.context_id = 1
a()
self.context_id = 2
a()
# Re-schedule context 1.
self.context_id = 1
a()
yappi.stop()
threadstats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(3, len(threadstats))
self.assertEqual(0, threadstats[0].id)
self.assertEqual(1, threadstats[1].id)
self.assertEqual(2, threadstats[2].id)
self.assertEqual(1, threadstats[0].sched_count)
self.assertEqual(2, threadstats[1].sched_count) # Context 1 ran twice.
self.assertEqual(1, threadstats[2].sched_count)
funcstats = yappi.get_func_stats()
self.assertEqual(4, utils.find_stat_by_name(funcstats, 'a').ncall)
def test_pause_resume(self):
yappi.set_context_id_callback(lambda: self.context_id)
yappi.set_clock_type('wall')
# Start in context 0.
self.context_id = 0
yappi.start()
time.sleep(0.08)
# Switch to context 1.
self.context_id = 1
time.sleep(0.05)
# Switch back to context 0.
self.context_id = 0
time.sleep(0.07)
yappi.stop()
t_stats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(2, len(t_stats))
self.assertEqual(0, t_stats[0].id)
self.assertEqual(2, t_stats[0].sched_count)
self.assertAlmostEqual(0.2, t_stats[0].ttot, places=2)
self.assertEqual(1, t_stats[1].id)
self.assertEqual(1, t_stats[1].sched_count)
# Context 1 was first scheduled 0.08 sec after context 0.
self.assertAlmostEqual(0.12, t_stats[1].ttot, places=2)
def teardown_profiler(self):
if not self.is_profiling():
return
path = os.environ["DRIVE_YAPPI"]
if not os.path.exists(path):
os.mkdir(path)
report_path = os.path.join(path, self.id() + '-yappi-threads')
with open(report_path, 'w') as fd:
columns = {0: ("name", 80), 1: ("tid", 15), 2: ("ttot", 8), 3: ("scnt", 10)}
yappi.get_thread_stats().print_all(out=fd, columns=columns)
report_path = os.path.join(path, self.id() + '-yappi-fcts')
with open(report_path, 'w') as fd:
columns = {0: ("name", 80), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)}
stats = yappi.get_func_stats()
stats.strip_dirs()
stats.print_all(out=fd, columns=columns)
log.debug("Profiler Report generated in '%s'", report_path)
def test_start_flags(self):
self.assertEqual(_yappi._get_start_flags(), None)
yappi.start()
def a(): pass
a()
self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0)
self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1)
self.assertEqual(len(yappi.get_thread_stats()), 1)
def prof_threads(self):
"""
Return profile threads info
"""
import yappi
i = yappi.get_thread_stats()
out = six.StringIO()
i.print_all(out=out)
return out.getvalue()
def main():
try:
if os.getenv('DEFAULT_SCORE_HOST') is not None:
os.system("ssh-keyscan " + os.getenv('DEFAULT_SCORE_HOST') +
" >> /root/.ssh/known_hosts")
if conf.ENABLE_PROFILING:
yappi.start()
launcher.main(sys.argv[1:])
yappi.stop()
else:
launcher.main(sys.argv[1:])
except KeyboardInterrupt:
if conf.ENABLE_PROFILING:
yappi.stop()
print('Yappi result (func stats) ======================')
yappi.get_func_stats().print_all()
print('Yappi result (thread stats) ======================')
yappi.get_thread_stats().print_all()
async def get_prof_threads():
"""
Get running threads info
:return:
"""
import yappi
i = yappi.get_thread_stats()
out = StringIO()
i.print_all(out=out)
return out.getvalue()
async def print_profiler_stats(event_loop, time_delay):
"""
Print profiler statistics after a number of seconds.
"""
try:
import yappi
except ImportError:
return
await asyncio.sleep(time_delay, loop=event_loop)
print("Printing profiler stats")
yappi.get_func_stats().print_all(out=sys.stdout,
columns={
0: ("name", 60),
1: ("ncall", 5),
2: ("tsub", 8),
3: ("ttot", 8),
4: ("tavg", 8)
})
yappi.get_func_stats().save(PROFILING_OUTPUT_FILE, type="callgrind")
yappi.get_thread_stats().print_all()
def test_subsequent_profile(self):
import threading
WORKER_COUNT = 5
def a(): pass
def b(): pass
def c(): pass
_timings = {"a_1":3,"b_1":2,"c_1":1,}
yappi.start()
def g(): pass
g()
yappi.stop()
yappi.clear_stats()
_yappi._set_test_timings(_timings)
yappi.start()
_dummy = []
for i in range(WORKER_COUNT):
t = threading.Thread(target=a)
t.start()
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=b)
t.start()
_dummy.append(t)
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=a)
t.start()
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=c)
t.start()
t.join()
yappi.stop()
yappi.start()
def f():
pass
f()
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, 'a')
fsb = utils.find_stat_by_name(stats, 'b')
fsc = utils.find_stat_by_name(stats, 'c')
self.assertEqual(fsa.ncall, 10)
self.assertEqual(fsb.ncall, 5)
self.assertEqual(fsc.ncall, 5)
self.assertEqual(fsa.ttot, fsa.tsub, 30)
self.assertEqual(fsb.ttot, fsb.tsub, 10)
self.assertEqual(fsc.ttot, fsc.tsub, 5)
# MACOSx optimizes by only creating one worker thread
self.assertTrue(len(yappi.get_thread_stats()) >= 2)
def test_callback(self):
self.context_id = 0
self.context_name = 'a'
yappi.set_context_id_callback(lambda: self.context_id)
yappi.set_context_name_callback(lambda: self.context_name)
yappi.start()
a()
self.context_id = 1
self.context_name = 'b'
a()
# Re-schedule context 0.
self.context_id = 0
self.context_name = 'a'
a()
yappi.stop()
threadstats = yappi.get_thread_stats().sort('name', 'ascending')
self.assertEqual(2, len(threadstats))
self.assertEqual(0, threadstats[0].id)
self.assertEqual('a', threadstats[0].name)
self.assertEqual(1, threadstats[1].id)
self.assertEqual('b', threadstats[1].name)
if __name__ == "__main__":
if DEBUG:
sys.argv.append("-h")
if TESTRUN:
import doctest
doctest.testmod()
if PROFILE:
import pstats
#Use yappi in multithreaded context
import yappi
yappi.start()
exit_code = main()
yappi.stop()
stats = yappi.get_func_stats()
tstats = yappi.get_thread_stats()
STAT_FILE = 'profile_stats{}'
stats.save(STAT_FILE.format('.pstat'), type='pstat')
with open(STAT_FILE.format('.txt'), 'w') as fh :
# pstat format better than yappi format
ps = pstats.Stats(STAT_FILE.format('.pstat'), stream = fh)
ps.sort_stats('time')
ps.print_stats()
ps.sort_stats('cumulative')
ps.print_stats()
#append thread stats
tstats.print_all(out = fh)
#add function stat with yappi format
stats.print_all(out=fh)
def lineReceived(self, line):
anon_tunnel = self.anon_tunnel
profile = self.profile
if line == 'threads':
for thread in threading.enumerate():
print "%s \t %d" % (thread.name, thread.ident)
elif line == 'p':
if profile:
for func_stats in yappi.get_func_stats().sort("subtime")[:50]:
print "YAPPI: %10dx %10.3fs" % (func_stats.ncall, func_stats.tsub), func_stats.name
else:
logger.error("Profiling disabled!")
elif line == 'P':
if profile:
filename = 'callgrindc_%d.yappi' % anon_tunnel.dispersy.lan_address[1]
yappi.get_func_stats().save(filename, type='callgrind')
else:
logger.error("Profiling disabled!")
elif line == 't':
if profile:
yappi.get_thread_stats().sort("totaltime").print_all()
else:
logger.error("Profiling disabled!")
elif line == 'c':
print "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype"
for circuit_id, circuit in anon_tunnel.community.circuits.items():
info_hash = circuit.info_hash.encode('hex')[:10] if circuit.info_hash else '?'
print "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % (circuit_id,
circuit.first_hop[0],
circuit.first_hop[1],
circuit.goal_hops,
len(circuit.hops),
circuit.bytes_down / 1024.0 / 1024.0,
circuit.bytes_up / 1024.0 / 1024.0,
info_hash,
circuit.ctype)
elif line.startswith('s'):
cur_path = os.getcwd()
line_split = line.split(' ')
filename = 'test_file' if len(line_split) == 1 else line_split[1]
if not os.path.exists(filename):
logger.info("Creating torrent..")
with open(filename, 'wb') as fp:
fp.write(os.urandom(50 * 1024 * 1024))
tdef = TorrentDef()
tdef.add_content(os.path.join(cur_path, filename))
tdef.set_tracker("udp://fake.net/announce")
tdef.set_private()
tdef.finalize()
tdef.save(os.path.join(cur_path, filename + '.torrent'))
else:
logger.info("Loading existing torrent..")
tdef = TorrentDef.load(filename + '.torrent')
logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10]))
defaultDLConfig = DefaultDownloadStartupConfig.getInstance()
dscfg = defaultDLConfig.copy()
dscfg.set_hops(1)
dscfg.set_dest_dir(cur_path)
anon_tunnel.session.lm.threadpool.call(0, anon_tunnel.session.start_download, tdef, dscfg)
elif line.startswith('i'):
# Introduce dispersy port from other main peer to this peer
line_split = line.split(' ')
to_introduce_ip = line_split[1]
to_introduce_port = int(line_split[2])
self.anon_tunnel.community.add_discovered_candidate(Candidate((to_introduce_ip, to_introduce_port), tunnel=False))
elif line.startswith('d'):
line_split = line.split(' ')
filename = 'test_file' if len(line_split) == 1 else line_split[1]
logger.info("Loading torrent..")
tdef = TorrentDef.load(filename + '.torrent')
logger.info("Loading torrent done")
defaultDLConfig = DefaultDownloadStartupConfig.getInstance()
dscfg = defaultDLConfig.copy()
dscfg.set_hops(1)
dscfg.set_dest_dir(os.path.join(os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port()))
def start_download():
def cb(ds):
logger.info('Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' %
(tdef.get_infohash().encode('hex')[:10],
ds.get_current_speed('down'),
ds.get_progress(),
dlstatus_strings[ds.get_status()],
sum(ds.get_num_seeds_peers()),
sum(1 for _ in anon_tunnel.community.dispersy_yield_verified_candidates())))
return 1.0, False
download = anon_tunnel.session.start_download(tdef, dscfg)
download.set_state_callback(cb, delay=1)
anon_tunnel.session.lm.threadpool.call(0, start_download)
elif line == 'q':
anon_tunnel.stop()
return
elif line == 'r':
print "circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)"
from_to = anon_tunnel.community.relay_from_to
for key in from_to.keys():
relay = from_to[key]
logger.info("%s-->\t%s\t\t%.2f" % ((key[0], key[1]), (relay.sock_addr, relay.circuit_id),
relay.bytes[1] / 1024.0 / 1024.0,))
import yappi
#yappi.set_clock_type("waLL")
#yappi.start(builtins=True)
yappi.start()
class A:
def bar(self):
pass
def foo():
def inner_foo():
pass
import time
time.sleep(2.0)
for i in range(20000000):
pass
a = A()
a.bar()
inner_foo()
foo()
#yappi.write_callgrind_stats()
yappi.get_func_stats().sort("totaLTiMe").print_all()
yappi.get_thread_stats().print_all()
#import cProfile
#cProfile.run('foo()', 'fooprof')
#import pstats
#p = pstats.Stats('fooprof')
#p.strip_dirs().sort_stats(-1).print_stats()
def run_and_get_thread_stats(func, *args, **kwargs):
run_with_yappi(func, *args, **kwargs)
return yappi.get_thread_stats()
def test_ctx_stats(self):
from threading import Thread
DUMMY_WORKER_COUNT = 5
yappi.start()
class DummyThread(Thread):
pass
def dummy():
pass
def dummy_worker():
pass
for i in range(DUMMY_WORKER_COUNT):
t = DummyThread(target=dummy_worker)
t.start()
t.join()
yappi.stop()
stats = yappi.get_thread_stats()
tsa = utils.find_stat_by_name(stats, "DummyThread")
self.assertTrue(tsa is not None)
yappi.clear_stats()
time.sleep(1.0)
_timings = {"a_1": 6, "b_1": 5, "c_1": 3, "d_1": 1, "a_2": 4, "b_2": 3, "c_2": 2, "d_2": 1}
_yappi._set_test_timings(_timings)
class Thread1(Thread):
pass
class Thread2(Thread):
pass
def a():
b()
def b():
c()
def c():
d()
def d():
time.sleep(0.6)
yappi.set_clock_type("wall")
yappi.start()
t1 = Thread1(target=a)
t1.start()
t2 = Thread2(target=a)
t2.start()
t1.join()
t2.join()
stats = yappi.get_thread_stats()
# the fist clear_stats clears the context table?
tsa = utils.find_stat_by_name(stats, "DummyThread")
self.assertTrue(tsa is None)
tst1 = utils.find_stat_by_name(stats, "Thread1")
tst2 = utils.find_stat_by_name(stats, "Thread2")
tsmain = utils.find_stat_by_name(stats, "_MainThread")
dummy() # call dummy to force ctx name to be retrieved again.
self.assertTrue(len(stats) == 3)
self.assertTrue(tst1 is not None)
self.assertTrue(tst2 is not None)
# TODO: I put dummy() to fix below, remove the comments after a while.
self.assertTrue( # FIX: I see this fails sometimes
tsmain is not None, 'Could not find "_MainThread". Found: %s' % (", ".join(utils.get_stat_names(stats)))
)
self.assertTrue(1.0 > tst2.ttot >= 0.5)
self.assertTrue(1.0 > tst1.ttot >= 0.5)
# test sorting of the ctx stats
stats = stats.sort("totaltime", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.ttot >= stat.ttot)
prev_stat = stat
stats = stats.sort("totaltime", "asc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.ttot <= stat.ttot)
prev_stat = stat
stats = stats.sort("schedcount", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.sched_count >= stat.sched_count)
prev_stat = stat
stats = stats.sort("name", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.name >= stat.name)
prev_stat = stat
self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sorttype_arg")
self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sortorder_arg")
def test_ctx_stats(self):
from threading import Thread
DUMMY_WORKER_COUNT = 5
yappi.start()
class DummyThread(Thread): pass
def dummy_worker():
pass
for i in range(DUMMY_WORKER_COUNT):
t = DummyThread(target=dummy_worker)
t.start()
t.join()
yappi.stop()
stats = yappi.get_thread_stats()
tsa = utils.find_stat_by_name(stats, "DummyThread")
self.assertTrue(tsa is not None)
yappi.clear_stats()
import time
time.sleep(1.0)
_timings = {"a_1":6,"b_1":5,"c_1":3, "d_1":1, "a_2":4,"b_2":3,"c_2":2, "d_2":1}
_yappi._set_test_timings(_timings)
class Thread1(Thread): pass
class Thread2(Thread): pass
def a():
b()
def b():
c()
def c():
d()
def d():
time.sleep(0.6)
yappi.set_clock_type("wall")
yappi.start()
t1 = Thread1(target=a)
t1.start()
t2 = Thread2(target=a)
t2.start()
t1.join()
t2.join()
stats = yappi.get_thread_stats()
# the fist clear_stats clears the context table?
tsa = utils.find_stat_by_name(stats, "DummyThread")
self.assertTrue(tsa is None)
tst1 = utils.find_stat_by_name(stats, "Thread1")
tst2 = utils.find_stat_by_name(stats, "Thread2")
tsmain = utils.find_stat_by_name(stats, "_MainThread")
#stats.print_all()
self.assertTrue(len(stats) == 3)
self.assertTrue(tst1 is not None)
self.assertTrue(tst2 is not None)
self.assertTrue(tsmain is not None) # I see this fails sometimes, probably
# because Py_ImportNoBlock() fails to import and get the thread class name
# sometimes.
self.assertTrue(1.0 > tst2.ttot >= 0.5)
self.assertTrue(1.0 > tst1.ttot >= 0.5)
# test sorting of the ctx stats
stats = stats.sort("totaltime", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.ttot >= stat.ttot)
prev_stat = stat
stats = stats.sort("totaltime", "asc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.ttot <= stat.ttot)
prev_stat = stat
stats = stats.sort("schedcount", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.sched_count >= stat.sched_count)
prev_stat = stat
stats = stats.sort("name", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.name >= stat.name)
prev_stat = stat
self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sorttype_arg")
self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sortorder_arg")
import yappi
def test_function():
pass
class T(Thread):
def __init__(self):
Thread.__init__(self)
def run(self): # takes about 5 seconds
for i in xrange(100):
self.test_method()
test_function()
def test_method(self):
sleep(random.random() / 10)
yappi.start()
#######################
threads = [T() for i in xrange(3)]
for t in threads:
t.start()
for i in xrange(100):
test_function()
for t in threads:
t.join()
#######################
yappi.stop()
yappi.get_func_stats().sort("totaltime").print_all()
yappi.get_thread_stats().sort("totaltime").print_all()
except KeyboardInterrupt:
if mpstate.settings.requireexit:
print("Interrupt caught. Use 'exit' to quit MAVProxy.")
#Just lost the map and console, get them back:
for (m,pm) in mpstate.modules:
if m.name in ["map", "console"]:
if hasattr(m, 'unload'):
try:
m.unload()
except Exception:
pass
reload(m)
m.init(mpstate)
else:
mpstate.status.exit = True
sys.exit(1)
if opts.profile:
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
#this loop executes after leaving the above loop and is for cleanup on exit
for (m,pm) in mpstate.modules:
if hasattr(m, 'unload'):
print("Unloading module %s" % m.name)
m.unload()
sys.exit(1)
def sys_exit_call(signum, frame):
print "profiling result:"
yappi.get_func_stats().print_all(columns={0:("name",100), 1:("ncall", 15),
2:("tsub", 8), 3:("ttot", 8), 4:("tavg",8)})
yappi.get_thread_stats().print_all()
sys.exit(1)
print("\nConnection successful!\n")
except Exception as err:
print("\nProblem connecting {0}\n ".format (err))
pass
sys.exit(0)
elif (cmdline_arg_isset("-yappi")):
print("Running with yappi profiler")
import yappi
yappi.start()
SAMPListener()
profiler = open("profile.data", "w")
yappi.get_func_stats().debug_print() #print_all()
yappi.get_thread_stats().print_all(out=profiler)
profiler.close()
else:
print("Starting logger")
options = read_options_from_commandline(None, ignore_errors=True)
options = podi_logging.setup_logging(options)
SAMPListener()
print("Shutting down QuickReduce logging")
podi_logging.shutdown_logging(options)
output_error("Specify scene file", output_function)
exit(1)
else:
scene_desc = pickle.load(sys.stdin)
pstd = PSTD(args.multi_threaded, args.write_plot, args.write_array, scene_desc, output_function)
out = sys.__stdout__
if args.file:
print("Writing to %s"%args.file)
out = open(args.file,'w')
out.write("openPSTD --- %s\n"%datetime.now().strftime('%d-%m-%y %H:%M'))
out.write("Profile output\n"
"Scene file %s\n"
"Parallel: %s\n"
"Plotting: %s\n"
"Writing arrays: %s\n\n"%(args.scene_file,args.multi_threaded,args.write_plot,args.write_array))
time_start = time.time()
yappi.start()
pstd.run()
yappi.stop()
time_passed = time.time() - time_start
out.write("Time passed: %.3f\n"%time_passed)
funcstats = yappi.get_func_stats()
funcstats.sort(sort_type='tsub', sort_order="desc")
funcstats.print_all(out=out,columns={0:("name",36), 1:("ncall", 8), 2:("tsub", 8), 3:("ttot", 8), 4:("tavg",8)})
yappi.get_thread_stats().print_all(out=out)
if args.file:
out.close()
exit()
# yappi.start()
# foo()