def test_profile_single_context(self): def id_callback(): return self.callback_count def a(): pass self.callback_count = 1 yappi.set_context_id_callback(id_callback) yappi.start(profile_threads=False) a() # context-id:1 self.callback_count = 2 a() # context-id:2 stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, "a") self.assertEqual(fsa.ncall, 1) yappi.stop() yappi.clear_stats() self.callback_count = 1 yappi.start() # profile_threads=True a() # context-id:1 self.callback_count = 2 a() # context-id:2 stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, "a") self.assertEqual(fsa.ncall, 2)
def test_filter(self): def a(): pass def b(): a() def c(): b() _TCOUNT = 5 ts = [] yappi.start() for i in range(_TCOUNT): t = threading.Thread(target=c) t.start() ts.append(t) for t in ts: t.join() yappi.stop() fstats = yappi.get_func_stats(filter={"ctx_id":9}) self.assertTrue(fstats.empty()) fstats = yappi.get_func_stats(filter={"ctx_id":0, "name":"c"}) # main thread self.assertTrue(fstats.empty()) for i in range(1, _TCOUNT): fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"a", "ncall":1}) self.assertEqual(fstats.pop().ncall, 1) fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"b"}) self.assertEqual(fstats.pop().ncall, 1) fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"c"}) self.assertEqual(fstats.pop().ncall, 1)
def test_merge_multithreaded_stats(self): import threading import _yappi timings = {"a_1":2, "b_1":1} _yappi._set_test_timings(timings) def a(): pass def b(): pass yappi.start() t = threading.Thread(target=a) t.start() t.join() t = threading.Thread(target=b) t.start() t.join() yappi.get_func_stats().save("ystats1.ys") yappi.clear_stats() _yappi._set_test_timings(timings) self.assertEqual(len(yappi.get_func_stats()), 0) self.assertEqual(len(yappi.get_thread_stats()), 1) t = threading.Thread(target=a) t.start() t.join() self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0) self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1) yappi.get_func_stats().save("ystats2.ys") stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys",]) fsa = utils.find_stat_by_name(stats, "a") fsb = utils.find_stat_by_name(stats, "b") self.assertEqual(fsa.ncall, 2) self.assertEqual(fsb.ncall, 1) self.assertEqual(fsa.tsub, fsa.ttot, 4) self.assertEqual(fsb.tsub, fsb.ttot, 1)
def stop(self): self._running = False self._tick_timer.cancel() self._stop_time = time.time() if self._app.args.yappi and USE_YAPPI: yappi.get_func_stats().print_all()
def stats_string(self, id): output = StringIO.StringIO() yappi.get_func_stats().print_all(out=output) m = ProfilerResponse() m.id = id m.ystats_string = output.getvalue() self.writer.addCommand(m)
def wrapped(*args, **kwargs): yappi.start() result = func(*args, **kwargs) yappi.stop() prof_file = "%s.%s" % (func.__name__, time.time()) #prof_file = "callgrind.a.1" yappi.get_func_stats().save(prof_file, "ystat") return result
def on_close(self, e): self.netcontroller.running = False #self.net_thread.quit() if self.args.profile: try: import yappi yappi.get_func_stats().print_all() except ImportError: pass
def update(): computer.run( clock.value ) if io.hasExited: clock.stop() print( 'See you later!' ) yappi.get_func_stats().print_all()
def cmd_stop(self, args): """stop profiling""" print "Profile results:" import yappi # We do the import here so that we won't barf if run normally and yappi not available yappi.get_func_stats().print_all( columns={0: ("name", 50), 1: ("ncall", 5), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8)} ) yappi.get_thread_stats().print_all() yappi.stop()
def yappi_prof_call(func, *args): ''' https://code.google.com/p/yappi/wiki/usageyappi_v092 ''' import yappi yappi.start() result = func(*args) yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() return result
def clusterizacao(request, debate_id, qtdGrupos=3): ## COLOCAR COMO OPÇÃO PARA O USUÁRIO SE ELE QUER AGRUPAR ## PELO POSICIONAMENTO INICIAL OU FINAL print "view-clusterização em funcionamento!!!" inicio = datetime.now() print inicio,"view clusterizacao" yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() auxResult = clusterArgInicial(debate_id) duration = time.time() - start stats = yappi.get_func_stats() stats.save('clusterArgInicial.out', type = 'callgrind') tese = auxResult[5] yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() # resultado = gruposArgumentacao(auxResult, qtdeGrupos=3, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=4, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=5, LSA=True, Normalizacao=True) # resultado = gruposArgumentacao(auxResult, qtdeGrupos=6, LSA=True, Normalizacao=True) resultado = gruposArgumentacao(auxResult, qtdeGrupos=int(qtdGrupos), LSA=None, Normalizacao=True) duration = time.time() - start stats = yappi.get_func_stats() stats.save('gruposArgumentacao.out', type = 'callgrind') grupo1 = resultado[0] grupo2 = resultado[1] grupo3 = resultado[2] grupo4 = resultado[3] grupo5 = resultado[4] grupo6 = resultado[5] context = RequestContext(request,{'results' : [grupo1,grupo2,grupo3,grupo4,\ len(grupo1),len(grupo2),len(grupo3),len(grupo4), tese, \ grupo5, len(grupo5), grupo6, len(grupo6)], 'grupo' : Grupo.objects.filter(idgrupo=1064)[0]}) return render(request, 'clusterizacao.html',context)
def main(): print('Main TID: {}'.format(gettid())) args = Bunch(channel=None, devices=[], generate=False, ui='main.ui') yappi.start() exit_value = epyqlib.__main__.main(args=args) yappi.stop() yappi.get_func_stats().save('yappi.stats', type='pstat') yappi.get_thread_stats().print_all() return exit_value
def run(): host = "http://localhost:8080" urls = ["/client.py", "/echo_server.py"] global all_task all_task = len(urls) yappi.start() for url in urls: fetch(host + url) print "fetch url:", url yappi.get_func_stats().print_all() IOLoop.instance().start()
def test_no_stats_different_clock_type_load(self): def a(): pass yappi.start() a() yappi.stop() yappi.get_func_stats().save("ystats1.ys") yappi.clear_stats() yappi.set_clock_type("WALL") yappi.start() yappi.stop() stats = yappi.get_func_stats().add("ystats1.ys") fsa = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa is not None)
def tearDown(self): fstats = yappi.get_func_stats() if not fstats._debug_check_sanity(): sys.stdout.write("ERR: Duplicates found in Func stats\r\n") fstats.debug_print() for fstat in fstats: if not fstat.children._debug_check_sanity(): sys.stdout.write("ERR: Duplicates found in ChildFunc stats\r\n") fstat.children.print_all() tstats = yappi.get_func_stats() if not tstats._debug_check_sanity(): sys.stdout.write("ERR: Duplicates found in Thread stats\r\n") tstats.print_all()
def main(argv): parser = argparse.ArgumentParser(description='Convert a FASTA and QUAL pair of files into a FASQ file') parser.add_argument('fasta_filename', help='the FASTA format input file path') parser.add_argument('qual_filename', help='the QUAL format input file path') parser.add_argument('-q', action='store_true', help="run quietly") parser.add_argument('-d', action='store_true', help="print diagnostics to stderr") parser.add_argument('-z','--gzip', action='store_true', help='input & output are gzipped') parser.add_argument('-p', '--profile', action='store_true', help="profile this execution") parser.add_argument('-o', '--output', default=sys.stdout, type=argparse.FileType('w'), help='the output FASTQ file path') args = parser.parse_args() global debug_flag debug_flag = args.d debug_flag and sys.stderr.write("%r\n" % args) if args.profile: import yappi # https://code.google.com/p/yappi/wiki/apiyappi yappi.start(builtins=True) if args.gzip: import gzip fasta = gzip.open(args.fasta_filename) qual = gzip.open(args.qual_filename) outf = gzip.GzipFile(fileobj=args.output, mode='wb') else: fasta = open(args.fasta_filename) qual = open(args.qual_filename) outf = args.output # alphabet irrelevant (and makes it slow) records = PairedFastaQualIterator(fasta, qual, alphabet=IUPAC.ambiguous_dna) records = PairedFastaQualIterator(fasta, qual) count = SeqIO.write(records, outf, "fastq") outf.close() if args.profile: yappi.get_thread_stats().print_all(sys.stderr) yappi.get_func_stats().sort("subtime").print_all(sys.stderr) print("Converted %i records" % count)
def test_basic(self): yappi.set_clock_type('wall') def dummy(): pass def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') dummy() # call dummy to force ctx name to be retrieved again. self.assertTrue(tsa is not None) # TODO: I put dummy() to fix below, remove the comments after a while. self.assertTrue( # FIX: I see this fails sometimes? tsm is not None, 'Could not find "_MainThread". Found: %s' % (', '.join(utils.get_stat_names(tstats))))
def stop_profiler(self): """ Stop yappi and write the stats to the output directory. Return the path of the yappi statistics file. """ if not self.profiler_running: raise RuntimeError("Profiler is not running") if not HAS_YAPPI: raise RuntimeError("Yappi cannot be found. Plase install the yappi library using your preferred package " "manager and restart Tribler afterwards.") yappi.stop() yappi_stats = yappi.get_func_stats() yappi_stats.sort("tsub") log_dir = os.path.join(self.session.config.get_state_dir(), 'logs') file_path = os.path.join(log_dir, 'yappi_%s.stats' % self.profiler_start_time) # Make the log directory if it does not exist if not os.path.exists(log_dir): os.makedirs(log_dir) yappi_stats.save(file_path, type='callgrind') yappi.clear_stats() self.profiler_running = False return file_path
def test_callback(self): self.context_id = 0 yappi.set_context_id_callback(lambda: self.context_id) yappi.start() a() self.context_id = 1 a() self.context_id = 2 a() # Re-schedule context 1. self.context_id = 1 a() yappi.stop() threadstats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(3, len(threadstats)) self.assertEqual(0, threadstats[0].id) self.assertEqual(1, threadstats[1].id) self.assertEqual(2, threadstats[2].id) self.assertEqual(1, threadstats[0].sched_count) self.assertEqual(2, threadstats[1].sched_count) # Context 1 ran twice. self.assertEqual(1, threadstats[2].sched_count) funcstats = yappi.get_func_stats() self.assertEqual(4, utils.find_stat_by_name(funcstats, 'a').ncall)
def test_basic(self): import threading import time yappi.set_clock_type('wall') def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') self.assertTrue(tsa is not None) self.assertTrue(tsm is not None) # FIX: I see this fails sometimes?
def test_singlethread_profiling(self): yappi.set_clock_type("wall") def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(profile_threads=False) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, "Worker1.a") fsa2 = utils.find_stat_by_name(stats, "a") self.assertTrue(fsa1 is None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa2.ttot > 0.1)
def test_print_formatting(self): def a(): pass def b(): a() func_cols = {1: ("name", 48), 0: ("ncall", 5), 2: ("tsub", 8)} thread_cols = {1: ("name", 48), 0: ("ttot", 8)} yappi.start() a() b() yappi.stop() fs = yappi.get_func_stats() cs = fs[1].children ts = yappi.get_thread_stats() # fs.print_all(out=sys.stderr, columns={1:("name", 70), }) # cs.print_all(out=sys.stderr, columns=func_cols) # ts.print_all(out=sys.stderr, columns=thread_cols) # cs.print_all(out=sys.stderr, columns={}) self.assertRaises(yappi.YappiError, fs.print_all, columns={1: ("namee", 9)}) self.assertRaises(yappi.YappiError, cs.print_all, columns={1: ("dd", 0)}) self.assertRaises(yappi.YappiError, ts.print_all, columns={1: ("tidd", 0)})
def yappi_logger(path): while True: time.sleep(60) yf = yappi.get_func_stats() they = sorted(yf, key=lambda x: x.ttot, reverse=True) #they = sorted(filter(lambda x: not x.builtin, yf), key=lambda x: x.ttot, reverse=True) opath = path + time.strftime('%Y%m%d_%H%M%S') + '.txt' with open(opath, 'wb') as fout: if True: sum_func_time = sum(map(lambda x: x.tsub, they)) fout.write('total {} seconds function time\n'.format(sum_func_time)) fout.write('func+subs\tfunc\ttot/call\tncalls\n') for x in they: fout.write(_yfstr(x)) else: yf.print_all( out=fout, columns={ 0:("name",80), 1:("ncall", 9), 2:("tsub", 8), 3:("ttot", 8), 4:("tavg",8), } ) sys.stderr.write('wrote profiling\n{}\n'.format(opath)) if os.path.exists('bobreak'): pdb.set_trace()
def main4(): import yappi yappi.start() main2() stats = yappi.get_func_stats() stats.sort('ttot') stats.print_all()
def test_children_stat_functions(self): _timings = {"a_1":5, "b_1":3, "c_1":1} _yappi._set_test_timings(_timings) def b(): pass def c(): pass def a(): b() c() yappi.start() a() b() # non-child call c() # non-child call stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'a') childs_of_a = fsa.children.get().sort("tavg", "desc") prev_item = None for item in childs_of_a: if prev_item: self.assertTrue(prev_item.tavg > item.tavg) prev_item = item childs_of_a.sort("name", "desc") prev_item = None for item in childs_of_a: if prev_item: self.assertTrue(prev_item.name > item.name) prev_item = item childs_of_a.clear() self.assertTrue(childs_of_a.empty())
def run_backend_and_or_ui(self): if self.args.start_frame is not None: print "fast-forwarding to frame %d..." % self.args.start_frame self.time_increment = 1. / self.args.frame_rate for n in range(self.args.start_frame): self._proceed_and_update() print "ok" else: self.entity.update(self.input) self.update() run_backend = not self.args.ui_only run_ui = not self.args.backend_only if run_ui: self._scene_class = self._entity_scene_module.Scene if self.args.with_profiler: import yappi yappi.start() if run_backend and run_ui: if self.args.websockets: websocket_server = self._create_websocket_server() self._start_in_new_thread(websocket_server) self._server = self._create_single_process_server() self._set_up_timed_refresh() self._start_in_new_thread(self._server) client = SingleProcessClient(self._server) self.run_ui(client) elif run_backend: self._server = self._create_websocket_server() self._set_up_timed_refresh() try: self._server.start() except KeyboardInterrupt: pass elif run_ui: if self.args.no_websockets: client = None else: client = WebsocketClient(self.args.backend_host) self.run_ui(client) if self.args.with_profiler: yappi.get_func_stats().print_all()
def test_lambda(self): f = lambda: time.sleep(0.3) yappi.set_clock_type("wall") yappi.start() f() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, "<lambda>") self.assertTrue(fsa.ttot > 0.1)
def _stop_profiling(filename, format): logging.debug("Stopping CPU profiling") with _lock: if yappi.is_running(): yappi.stop() stats = yappi.get_func_stats() stats.save(filename, format) yappi.clear_stats()
def stop(self): if not yappi.is_running(): raise UsageError("CPU profiler is not running") logging.info("Stopping CPU profiling") yappi.stop() stats = yappi.get_func_stats() stats.save(self.filename, self.format) yappi.clear_stats()
def test_generator(self): def _gen(n): while(n > 0): yield n n -= 1 yappi.start() for x in _gen(5): pass self.assertTrue(yappi.convert2pstats(yappi.get_func_stats()) is not None)
async def on_message(message): Bot_Prefix = "+" if getenv("PRODUCTION") is None: Bot_Prefix = "-" if not message.content.startswith(Bot_Prefix): return # Split input args = message.content[len(Bot_Prefix):].split() if len(args) == 0: return cmd = args[0].lower() # the arg array ex. ["hello", "world"] args = args[1:] s = message.channel.send if (cmd == "8" or cmd == "report" or cmd == "define" or cmd == "stars" or cmd == "homepage" or cmd == "clapify" or cmd == "cookie" or cmd == "say") and Preconditions.args_are_valid(args): return await s(embed=EmbedUtil.prep( "That command expected an argument (or arguments), but you didn't give it any!", "[Read the docs?](https://cakebot.club/docs/commands/)", )) tcu_result = TextCommandsUtil.handle_common_commands(args, cmd) if tcu_result != "": return await s(tcu_result) if cmd == "help": return await s(embed=EmbedUtil.prep( title="Help", description= "You can check out [this page of our website](https://cakebot.club/docs/commands/) for a full command list!", )) elif cmd == "ping": return await s(f"ЪЈЊ - websocket responded in {client.latency}") elif cmd == "invite": return await s(embed=EmbedUtil.prep( "Invite Cakebot", f"[Click here to invite me!]({oauth_url(580573141898887199, permissions=discord.Permissions.all())})", )) elif cmd == "info": return await s( TextCommandsUtil.data_template.format( message.guild.name, str(message.guild.owner), len(message.guild.members), message.guild.region, message.guild.id, message.guild.premium_subscription_count, str(message.guild.is_icon_animated()), str(message.guild.created_at), str(message.guild.large), str(message.guild.mfa_level == 1), )) elif cmd == "report": return await GitHubUtil.report(s, g, args, message) elif cmd == "iss": m = await s("Calculating...") imp = IssApi.IssLocater() lat = imp.lat lon = imp.lon from reverse_geocoder import search geodata = search((lat, lon)) location = "{0}, {1}".format(geodata[0]["admin1"], geodata[0]["cc"]) await m.delete() return await s(embed=EmbedUtil.prep( "International Space Station", "Where it is right now!").add_field( name="Location above Earth", value=str(location), inline=False).add_field( name="Latitude", value=str(lat), inline=False).add_field( name="Longitude", value=str(lon), inline=False)) elif cmd == "fact": return await s(embed=EmbedUtil.prep("Random Fact", FactImp().fact())) elif cmd == "slots": slotz = result() top = row() btm = row() form = "win" if slotz[0] == 1 else "lose" return await s( f"Рађ{top[0]}{top[1]}{top[2]}\n" # the line above contains unicode, DO NOT REMOVE + f"**>** {slotz[1][0]}{slotz[1][1]}{slotz[1][2]} **<**\n" + f" {btm[0]}{btm[1]}{btm[2]}" + f"\n**You {form}!**") elif cmd == "reboot": if message.author.id in UserUtil.admins(): await s("Restarting. This may take up to 5 minutes.") # make the bot crash, forcing our server to turn it back on _exit(1) else: return await s(":x: **You are not authorized to run this!**") elif cmd == "stars": try: return await s( f"`{args[0]}` has *{g.get_repo(args[0]).stargazers_count}* stars." ) except: return await s( "Failed to get count. Is the repository valid and public?") elif cmd == "homepage": try: url = g.get_repo(args[0]).homepage if url is None: url = "(error: homepage not specified by owner)" return await s(f"{args[0]}'s homepage is located at {url}") except: return await s( "Failed to fetch homepage. Is the repository valid and public?" ) elif cmd == "boomer": return await s(file=discord.File("content/boomer.jpeg")) elif cmd == "cookie" or cmd == "cookies": subcommand = args[0] args = args[1:] userId = TextCommandsUtil.get_mentioned_id(args) if subcommand in ["balance", "bal"]: count = 0 if userId == 0: # assume user wants themself count = Database.get_count(message.author.id, config) else: count = Database.get_count(userId, config) return await s(embed=EmbedUtil.prep( title="Cookies", description=f"User has {count} cookies.", )) elif subcommand in ["give", "to"]: if userId == 0: return await s( "I don't see who I should give the cookie to. Try mentioning them." ) new_count = Database.add_cookie(userId, config) return await s( f"Gave <@!{userId}> a cookie. They now have {new_count} cookies." ) elif cmd == "define": if wordsapi_token is None: return await s( "This command is disabled due to a configuration error on my host's end - didn't find a WordsAPI token in the config!" ) return await s(embed=TextCommandsUtil.define(args, wordsapi_token)) elif cmd == "start-profiler": if message.author.id in UserUtil.admins(): await s( "Started the profiler. Once you are done, run stop-profiler.") yappi.set_clock_type("wall") yappi.start() else: return await s(":x: **You are not authorized to run this!**") elif cmd == "stop-profiler": if message.author.id in UserUtil.admins(): await s("Saved profiler results to `profile.txt`.") yappi.stop() yappi.get_func_stats().print_all(open("profile.txt", "w")) else: return await s(":x: **You are not authorized to run this!**")
def _dump_profile(): import yappi yappi.get_func_stats().save('master_prof.out', type='pstat')
def lineReceived(self, line): anon_tunnel = self.anon_tunnel profile = self.profile if line == 'threads': for thread in threading.enumerate(): print "%s \t %d" % (thread.name, thread.ident) elif line == 'p': if profile: for func_stats in yappi.get_func_stats().sort("subtime")[:50]: print "YAPPI: %10dx %10.3fs" % ( func_stats.ncall, func_stats.tsub), func_stats.name else: logger.error("Profiling disabled!") elif line == 'P': if profile: filename = 'callgrindc_%d.yappi' % anon_tunnel.dispersy.lan_address[ 1] yappi.get_func_stats().save(filename, type='callgrind') else: logger.error("Profiling disabled!") elif line == 't': if profile: yappi.get_thread_stats().sort("totaltime").print_all() else: logger.error("Profiling disabled!") elif line == 'c': print "========\nCircuits\n========\nid\taddress\t\t\t\t\tgoal\thops\tIN (MB)\tOUT (MB)\tinfohash\ttype" for circuit_id, circuit in anon_tunnel.community.circuits.items(): info_hash = circuit.info_hash.encode( 'hex')[:10] if circuit.info_hash else '?' print "%d\t%s:%d\t%d\t%d\t\t%.2f\t\t%.2f\t\t%s\t%s" % ( circuit_id, circuit.first_hop[0], circuit.first_hop[1], circuit.goal_hops, len(circuit.hops), circuit.bytes_down / 1024.0 / 1024.0, circuit.bytes_up / 1024.0 / 1024.0, info_hash, circuit.ctype) elif line.startswith('s'): cur_path = os.getcwd() line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] if not os.path.exists(filename): logger.info("Creating torrent..") with open(filename, 'wb') as fp: fp.write(os.urandom(50 * 1024 * 1024)) tdef = TorrentDef() tdef.add_content(os.path.join(cur_path, filename)) tdef.set_tracker("udp://fake.net/announce") tdef.set_private() tdef.finalize() tdef.save(os.path.join(cur_path, filename + '.torrent')) else: logger.info("Loading existing torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("loading torrent done, infohash of torrent: %s" % (tdef.get_infohash().encode('hex')[:10])) defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir(cur_path) anon_tunnel.session.lm.threadpool.call( 0, anon_tunnel.session.start_download, tdef, dscfg) elif line.startswith('i'): # Introduce dispersy port from other main peer to this peer line_split = line.split(' ') to_introduce_ip = line_split[1] to_introduce_port = int(line_split[2]) self.anon_tunnel.community.add_discovered_candidate( Candidate((to_introduce_ip, to_introduce_port), tunnel=False)) elif line.startswith('d'): line_split = line.split(' ') filename = 'test_file' if len(line_split) == 1 else line_split[1] logger.info("Loading torrent..") tdef = TorrentDef.load(filename + '.torrent') logger.info("Loading torrent done") defaultDLConfig = DefaultDownloadStartupConfig.getInstance() dscfg = defaultDLConfig.copy() dscfg.set_hops(1) dscfg.set_dest_dir( os.path.join( os.getcwd(), 'downloader%s' % anon_tunnel.session.get_dispersy_port())) def start_download(): def cb(ds): logger.info( 'Download infohash=%s, down=%s, progress=%s, status=%s, seedpeers=%s, candidates=%d' % (tdef.get_infohash().encode('hex')[:10], ds.get_current_speed('down'), ds.get_progress(), dlstatus_strings[ds.get_status()], sum(ds.get_num_seeds_peers()), sum(1 for _ in anon_tunnel.community. dispersy_yield_verified_candidates()))) return 1.0, False download = anon_tunnel.session.start_download(tdef, dscfg) download.set_state_callback(cb, delay=1) anon_tunnel.session.lm.threadpool.call(0, start_download) elif line == 'q': anon_tunnel.stop() return elif line == 'r': print "circuit\t\t\tdirection\tcircuit\t\t\tTraffic (MB)" from_to = anon_tunnel.community.relay_from_to for key in from_to.keys(): relay = from_to[key] logger.info("%s-->\t%s\t\t%.2f" % ( (key[0], key[1]), (relay.sock_addr, relay.circuit_id), relay.bytes[1] / 1024.0 / 1024.0, ))
ICPol.ICPol.PROP_CONF = c * 0.25 yappi.start() graphs = [] for i in range(len(GRAPH_LIST)): G = GetGraph(GRAPH_LIST[i]) icPol = ICPol.ICPol(G) FILENAME = "Result/" + datetime.datetime.now().strftime( "%Y-%m-%d %H:%M:%S") DoExperiment(N_EXP, icPol, i) with open(FILENAME + "/Information.txt", 'w') as outfile: outfile.write(ExperimentInfo(icPol, i)) outfile.write('\nYAPPI LOG\n---------') stat = yappi.get_func_stats().print_all(out=outfile) yappi.stop() ICPol.ICPol.C_INC_MODE = 'logistic' for c in range(5): ICPol.ICPol.ACC_CONF = c * 0.25 ICPol.ICPol.PROP_CONF = 0.5 yappi.start() graphs = [] for i in range(len(GRAPH_LIST)): G = GetGraph(GRAPH_LIST[i]) icPol = ICPol.ICPol(G)
if args.verb: print('Full logging...') logging.basicConfig(level=logging.DEBUG) else: print('Error only logging...') logging.basicConfig(level=logging.ERROR) if args.xprof: import yappi import time yappi.set_clock_type("cpu") yappi.start() asyncio.run( local_benchmark.main_perf(messages_num=args.paym, wait_num=args.wait, verbose=args.verb)) if args.xprof: columns = { 0: ("name", 100), 1: ("ncall", 20), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8) } yappi.get_func_stats().strip_dirs().print_all(columns=columns) yappi.get_thread_stats().print_all()
def run_and_get_func_stats(func, *args, **kwargs): run_with_yappi(func, *args, **kwargs) return yappi.get_func_stats()
# Requires yappi to be installed, use easy_install yappi import yappi import sys from time import time from tribler import run if __name__ == '__main__': t1 = time() yappi.start() run() yappi.stop() print >> sys.stderr, "YAPPI: %s tribler has run for %s seconds" % \ (yappi.get_clock_type(), time() - t1) yappi_stats = yappi.get_func_stats() yappi_stats.sort("tsub") count = 0 for func_stat in yappi_stats: print >> sys.stderr, "YAPPI: %10dx %10.3fs %s" % \ (func_stat.ncall, func_stat.tsub, func_stat.name) count += 1 if count >= 50: break
def run_from_argv(self, argv): """ Execute command. Usually from script if __name__ == "__main__": import sys sys.exit(Command.run_from_argv()) """ parser = self.create_parser() self.add_default_arguments(parser) self.add_arguments(parser) options = parser.parse_args(argv) cmd_options = vars(options) args = cmd_options.pop("args", ()) loglevel = cmd_options.pop("loglevel") if loglevel: self.setup_logging(loglevel) enable_profiling = cmd_options.pop("enable_profiling", False) show_metrics = cmd_options.pop("show_metrics", False) show_usage = cmd_options.pop("show_usage", False) self.no_progressbar = cmd_options.pop("no_progressbar", False) if enable_profiling: # Start profiler import yappi yappi.start() try: if show_usage: import resource start_usage = resource.getrusage(resource.RUSAGE_SELF) return self.handle(*args, **cmd_options) or 0 except CommandError as e: self.print(str(e)) return 1 except KeyboardInterrupt: self.print("Ctrl+C") return 3 except AssertionError as e: if e.args and e.args[0]: self.print("ERROR: %s" % e.args[0]) else: self.print("Assertion error: %s" % e) return 4 except Exception: from noc.core.debug import error_report error_report() return 2 finally: if show_usage: stop_usage = resource.getrusage(resource.RUSAGE_SELF) self.show_usage(start_usage, stop_usage) if enable_profiling: i = yappi.get_func_stats() i.print_all( out=self.stdout, columns={ 0: ("name", 80), 1: ("ncall", 10), 2: ("tsub", 8), 3: ("ttot", 8), 4: ("tavg", 8), }, ) if show_metrics: from noc.core.perf import apply_metrics d = apply_metrics({}) self.print("Internal metrics:") for k in d: self.print("%40s : %s" % (k, d[k]))
func() def func3(): result = [] a = 0 for i in range(LARGE_NUMBER): a += i result.append(a) def main(): t = threading.Thread(target=func) t2 = threading.Thread(target=func2) t3 = threading.Thread(target=func3) t.start() t2.start() t3.start() t.join() t2.join() t3.join() if __name__ == '__main__': yappi.start() main() yappi.get_func_stats().strip_dirs().print_all() yappi.get_thread_stats().print_all()
def main(args, datasets): do_prof = args.do_profile do_tensorflow = not args.disable_tensorflow #use the environment variable for cupy/cuda choice args.use_cuda = USE_CUPY analysis_corrections = None if "analyze" in args.action: analysis_corrections = AnalysisCorrections(args, do_tensorflow) # Optionally disable pinned memory (will be somewhat slower) if args.use_cuda: import cupy if not args.pinned: cupy.cuda.set_allocator(None) cupy.cuda.set_pinned_memory_allocator(None) #Use sync-only datasets if args.do_sync: datasets = datasets_sync #Filter datasets by era datasets_to_process = [] for ds in datasets: if args.datasets is None or ds[0] in args.datasets: if args.eras is None or ds[1] in args.eras: datasets_to_process += [ds] print("Will consider dataset", ds) if len(datasets) == 0: raise Exception("No datasets considered, please check the --datasets and --eras options") datasets = datasets_to_process hmumu_utils.NUMPY_LIB, hmumu_utils.ha = choose_backend(args.use_cuda) Dataset.numpy_lib = hmumu_utils.NUMPY_LIB NUMPY_LIB = hmumu_utils.NUMPY_LIB # All analysis definitions (cut values etc) should go here analysis_parameters = { "baseline": { "nPV": 0, "NdfPV": 4, "zPV": 24, # Will be applied with OR "hlt_bits": { "2016": ["HLT_IsoMu24", "HLT_IsoTkMu24"], "2017": ["HLT_IsoMu27"], "2018": ["HLT_IsoMu24"], }, "muon_pt": 20, "muon_pt_leading": {"2016": 26.0, "2017": 29.0, "2018": 26.0}, "muon_eta": 2.4, "muon_iso": 0.25, "muon_id": {"2016": "medium", "2017": "medium", "2018": "medium"}, "muon_trigger_match_dr": 0.1, "muon_iso_trigger_matched": 0.15, "muon_id_trigger_matched": {"2016": "tight", "2017": "tight", "2018": "tight"}, "do_rochester_corrections": True, "do_lepton_sf": True, "do_jec": True, "jec_tag": {"2016": "Summer16_07Aug2017_V11", "2017": "Fall17_17Nov2017_V32", "2018": "Autumn18_V16"}, "jet_mu_dr": 0.4, "jet_pt_leading": {"2016": 35.0, "2017": 35.0, "2018": 35.0}, "jet_pt_subleading": {"2016": 25.0, "2017": 25.0, "2018": 25.0}, "jet_eta": 4.7, "jet_id": "tight", "jet_puid": "loose", "jet_veto_eta": [2.65, 3.139], "jet_veto_raw_pt": 50.0, "jet_btag": {"2016": 0.6321, "2017": 0.4941, "2018": 0.4184}, "do_factorized_jec": args.do_factorized_jec, "softjet_pt": 5.0, "softjet_evt_dr2": 0.04, "cat5_dijet_inv_mass": 400.0, "cat5_abs_jj_deta_cut": 2.5, "masswindow_z_peak": [76, 106], "masswindow_h_sideband": [110, 150], "masswindow_h_peak": [115, 135], "inv_mass_bins": 41, "extra_electrons_pt": 20, "extra_electrons_eta": 2.5, "extra_electrons_iso": 0.4, #Check if we want to apply this "extra_electrons_id": "mvaFall17V1Iso_WP90", "save_dnn_vars": True, "dnn_vars_path": "{0}/dnn_vars".format(args.out), #If true, apply mjj > cut, otherwise inverse "vbf_filter_mjj_cut": 350, "vbf_filter": { "dy_m105_160_mg": True, "dy_m105_160_amc": True, "dy_m105_160_vbf_mg": False, "dy_m105_160_vbf_amc": False, }, "ggh_nnlops_reweight": { "ggh_amc": 1, "ggh_amcPS": 1, "ggh_amcPS_TuneCP5down": 1, "ggh_amcPS_TuneCP5up": 1, "ggh_powheg": 2, "ggh_powhegPS": 2, }, "ZpT_reweight": { "2016": { "dy_0j": 2, "dy_1j": 2, "dy_2j": 2, "dy_m105_160_amc": 2, "dy_m105_160_vbf_amc": 2, }, "2017": { "dy_0j": 1, "dy_1j": 1, "dy_2j": 1, "dy_m105_160_amc": 1, "dy_m105_160_vbf_amc": 1, }, "2018": { "dy_0j": 1, "dy_1j": 1, "dy_2j": 1, "dy_m105_160_amc": 1, "dy_m105_160_vbf_amc": 1, }, }, #Pisa Group's DNN input variable order for keras "dnnPisa_varlist1_order": ['Mqq_log','Rpt','qqDeltaEta','ll_zstar','NSoft5','minEtaHQ','Higgs_pt','log(Higgs_pt)','Higgs_eta','Mqq','QJet0_pt_touse','QJet1_pt_touse','QJet0_eta','QJet1_eta','QJet0_phi','QJet1_phi','QJet0_qgl','QJet1_qgl'], "dnnPisa_varlist2_order": ['Higgs_m','Higgs_mRelReso','Higgs_mReso'], #Irene's DNN input variable order for keras "dnn_varlist_order": ['softJet5', 'dRmm','dEtamm','M_jj','pt_jj','eta_jj','phi_jj','M_mmjj','eta_mmjj','phi_mmjj','dEta_jj','Zep','dRmin_mj', 'dRmax_mj', 'dRmin_mmj','dRmax_mmj','dPhimm','leadingJet_pt','subleadingJet_pt', 'leadingJet_eta','subleadingJet_eta','leadingJet_qgl','subleadingJet_qgl','cthetaCS','Higgs_pt','Higgs_eta','Higgs_mass'], "dnn_input_histogram_bins": { "softJet5": (0,10,10), "dRmm": (0,5,11), "dEtamm": (-2,2,11), "dPhimm": (-2,2,11), "M_jj": (0,2000,11), "pt_jj": (0,400,11), "eta_jj": (-5,5,11), "phi_jj": (-5,5,11), "M_mmjj": (0,2000,11), "eta_mmjj": (-3,3,11), "phi_mmjj": (-3,3,11), "dEta_jj": (-3,3,11), "Zep": (-2,2,11), "dRmin_mj": (0,5,11), "dRmax_mj": (0,5,11), "dRmin_mmj": (0,5,11), "dRmax_mmj": (0,5,11), "leadingJet_pt": (0, 200, 11), "subleadingJet_pt": (0, 200, 11), "leadingJet_eta": (-5, 5, 11), "subleadingJet_eta": (-5, 5, 11), "leadingJet_qgl": (0, 1, 11), "subleadingJet_qgl": (0, 1, 11), "cthetaCS": (-1, 1, 11), "Higgs_pt": (0, 200, 11), "Higgs_eta": (-3, 3, 11), "Higgs_mass": (110, 150, 11), "dnn_pred": (0, 1, 1001), "dnn_pred2": (0, 1, 11), "bdt_ucsd": (-1, 1, 11), "bdt2j_ucsd": (-1, 1, 11), "bdt01j_ucsd": (-1, 1, 11), "MET_pt": (0, 200, 11), "hmmthetacs": (-1, 1, 11), "hmmphics": (-4, 4, 11), }, "categorization_trees": {}, "do_bdt_ucsd": True, "do_dnn_pisa": True, }, } histo_bins = { "muon_pt": np.linspace(0, 200, 101, dtype=np.float32), "muon_eta": np.linspace(-2.5, 2.5, 21, dtype=np.float32), "npvs": np.linspace(0, 100, 101, dtype=np.float32), "dijet_inv_mass": np.linspace(0, 2000, 11, dtype=np.float32), "inv_mass": np.linspace(70, 150, 11, dtype=np.float32), "numjet": np.linspace(0, 10, 11, dtype=np.float32), "jet_pt": np.linspace(0, 300, 101, dtype=np.float32), "jet_eta": np.linspace(-4.7, 4.7, 11, dtype=np.float32), "pt_balance": np.linspace(0, 5, 11, dtype=np.float32), "numjets": np.linspace(0, 10, 11, dtype=np.float32), "jet_qgl": np.linspace(0, 1, 11, dtype=np.float32), "higgs_inv_mass_uncertainty": np.linspace(0, 10, 101, dtype=np.float32), "higgs_rel_inv_mass_uncertainty": np.linspace(0, 0.05, 101, dtype=np.float32) } for hname, bins in analysis_parameters["baseline"]["dnn_input_histogram_bins"].items(): histo_bins[hname] = np.linspace(bins[0], bins[1], bins[2], dtype=np.float32) for masswindow in ["z_peak", "h_peak", "h_sideband"]: mw = analysis_parameters["baseline"]["masswindow_" + masswindow] histo_bins["inv_mass_{0}".format(masswindow)] = np.linspace(mw[0], mw[1], 41, dtype=np.float32) histo_bins["dnn_pred2"] = { "h_peak": np.array([0., 0.905, 0.915, 0.925, 0.935, 0.94, 0.945, 0.95, 0.955, 0.96, 0.965,0.97, 0.975,0.98, 0.985,1.0], dtype=np.float32), "z_peak": np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 1.0], dtype=np.float32), "h_sideband": np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0], dtype=np.float32), } analysis_parameters["baseline"]["histo_bins"] = histo_bins #analysis_parameters["oldjec"] = copy.deepcopy(analysis_parameters["baseline"]) #analysis_parameters["oldjec"]["jec_tag"]["2018"] = "Autumn18_V8" #Run baseline analysis outpath = "{0}/partial_results".format(args.out) try: os.makedirs(outpath) except FileExistsError as e: pass with open('{0}/parameters.pkl'.format(outpath), 'wb') as handle: pickle.dump(analysis_parameters, handle, protocol=pickle.HIGHEST_PROTOCOL) #Recreate dump of all filenames cache_filename = args.cache_location + "/datasets.json" if ("cache" in args.action) and (args.jobfiles is None): print("--action cache and no jobfiles specified, creating datasets.json dump of all filenames") if not os.path.isdir(args.cache_location): os.makedirs(args.cache_location) filenames_cache = {} for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset filenames_all = glob.glob(args.datapath + dataset_globpattern, recursive=True) filenames_all = [fn for fn in filenames_all if not "Friend" in fn] filenames_cache[dataset_name + "_" + dataset_era] = [ fn.replace(args.datapath, "") for fn in filenames_all] if len(filenames_all) == 0: raise Exception("Dataset {0} matched 0 files from glob pattern {1}, verify that the data files are located in {2}".format( dataset_name, dataset_globpattern, args.datapath )) #save all dataset filenames to a json file print("Creating a json dump of all the dataset filenames based on data found in {0}".format(args.datapath)) if os.path.isfile(cache_filename): print("Cache file {0} already exists, we will not overwrite it to be safe.".format(cache_filename), file=sys.stderr) print("Delete it or change --cache-location and try again.", file=sys.stderr) sys.exit(1) with open(cache_filename, "w") as fi: fi.write(json.dumps(filenames_cache, indent=2)) if ("cache" in args.action or "analyze" in args.action) and (args.jobfiles is None): #Create a list of job files for processing jobfile_data = [] print("Loading list of filenames from {0}".format(cache_filename)) if not os.path.isfile(cache_filename): raise Exception("Cached dataset list of filenames not found in {0}, please run this code with --action cache".format( cache_filename)) filenames_cache = json.load(open(cache_filename, "r")) for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset try: filenames_all = filenames_cache[dataset_name + "_" + dataset_era] except KeyError as e: print("Could not load {0} from {1}, please make sure this dataset has been added to cache".format( dataset_name + "_" + dataset_era, cache_filename), file=sys.stderr) raise e filenames_all_full = [args.datapath + "/" + fn for fn in filenames_all] chunksize = args.chunksize * chunksize_multiplier.get(dataset_name, 1) print("Saving dataset {0}_{1} with {2} files in {3} files per chunk to jobfiles".format( dataset_name, dataset_era, len(filenames_all_full), chunksize)) jobfile_dataset = create_dataset_jobfiles(dataset_name, dataset_era, filenames_all_full, is_mc, chunksize, args.out) jobfile_data += jobfile_dataset print("Dataset {0}_{1} consists of {2} chunks".format( dataset_name, dataset_era, len(jobfile_dataset))) assert(len(jobfile_data) > 0) assert(len(jobfile_data[0]["filenames"]) > 0) #For each dataset, find out which chunks we want to process if "cache" in args.action or "analyze" in args.action: jobfile_data = [] if not (args.jobfiles_load is None): args.jobfiles = [l.strip() for l in open(args.jobfiles_load).readlines()] if args.jobfiles is None: print("You did not specify to process specific dataset chunks, assuming you want to process all chunks") print("If this is not true, please specify e.g. --jobfiles data_2018_0.json data_2018_1.json ...") args.jobfiles = [] for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset jobfiles_dataset = glob.glob(args.out + "/jobfiles/{0}_{1}_*.json".format(dataset_name, dataset_era)) assert(len(jobfiles_dataset) > 0) if args.maxchunks > 0: jobfiles_dataset = jobfiles_dataset[:args.maxchunks] args.jobfiles += jobfiles_dataset #Now load the jobfiles assert(len(args.jobfiles) > 0) print("You specified --jobfiles {0}, processing only these dataset chunks".format(" ".join(args.jobfiles))) jobfile_data = [] for f in args.jobfiles: jobfile_data += [json.load(open(f))] chunkstr = " ".join(["{0}_{1}_{2}".format( ch["dataset_name"], ch["dataset_era"], ch["dataset_num_chunk"]) for ch in jobfile_data]) print("Will process {0} dataset chunks: {1}".format(len(jobfile_data), chunkstr)) assert(len(jobfile_data) > 0) #Start the profiler only in the actual data processing if do_prof: import yappi filename = 'analysis.prof' yappi.set_clock_type('cpu') yappi.start(builtins=True) if "cache" in args.action: print("Running the 'cache' step of the analysis, ROOT files will be opened and branches will be uncompressed") print("Will retrieve dataset filenames based on existing ROOT files on filesystem in datapath={0}".format(args.datapath)) try: os.makedirs(cmdline_args.cache_location) except Exception as e: pass run_cache(args, outpath, jobfile_data, analysis_parameters) if "analyze" in args.action: run_analysis(args, outpath, jobfile_data, analysis_parameters, analysis_corrections) if "merge" in args.action: with ProcessPoolExecutor(max_workers=args.nthreads) as executor: for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset fut = executor.submit(merge_partial_results, dataset_name, dataset_era, outpath) print("done merging") if do_prof: stats = yappi.get_func_stats() stats.save(filename, type='callgrind') import resource total_memory = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss total_memory += resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print("maxrss={0} MB".format(total_memory/1024))
def _on_keyboard_down(self, _keyboard, keycode, _text, modifiers): self.last_key_down = keycode ctrl_pressed = "ctrl" in modifiers if self.controls.note.focus: return # when making notes, don't allow keyboard shortcuts popup = self.popup_open if popup: if keycode[1] in ["f5", "f6", "f7", "f8"]: # switch between popups popup.dismiss() return elif keycode[1] in ["enter", "numpadenter"]: fn = getattr(popup.content, "on_submit", None) if fn: fn() return else: return shift_pressed = "shift" in modifiers shortcuts = self.shortcuts if keycode[1] == "spacebar": self.toggle_continuous_analysis() elif keycode[1] == "k": self.board_gui.toggle_coordinates() elif keycode[1] in ["pause", "break", "f15"] and not ctrl_pressed: self.controls.timer.paused = not self.controls.timer.paused elif keycode[1] in ["`", "~", "f12"]: self.zen = (self.zen + 1) % 3 elif keycode[1] in ["left", "z"]: self("undo", 1 + shift_pressed * 9 + ctrl_pressed * 9999) elif keycode[1] in ["right", "x"]: self("redo", 1 + shift_pressed * 9 + ctrl_pressed * 9999) elif keycode[1] == "home": self("undo", 9999) elif keycode[1] == "end": self("redo", 9999) elif keycode[1] == "pageup": self.controls.move_tree.make_selected_node_main_branch() elif keycode[1] == "n" and not ctrl_pressed: self("find-mistake", "undo" if shift_pressed else "redo") elif keycode[1] == "delete" and ctrl_pressed: self.controls.move_tree.delete_selected_node() elif keycode[1] == "c" and not ctrl_pressed: self.controls.move_tree.toggle_selected_node_collapse() elif keycode[1] == "n" and ctrl_pressed: self("new-game-popup") elif keycode[1] == "l" and ctrl_pressed: self("analyze-sgf-popup") elif keycode[1] == "s" and ctrl_pressed: self("save-game") elif keycode[1] == "d" and ctrl_pressed: self("save-game-as-popup") elif keycode[1] == "c" and ctrl_pressed: Clipboard.copy(self.game.root.sgf()) self.controls.set_status(i18n._("Copied SGF to clipboard."), STATUS_INFO) elif keycode[1] == "v" and ctrl_pressed: self.load_sgf_from_clipboard() elif keycode[1] in shortcuts.keys() and not ctrl_pressed: shortcut = shortcuts[keycode[1]] if isinstance(shortcut, Widget): shortcut.trigger_action(duration=0) else: self(*shortcut) elif keycode[1] == "f9" and self.debug_level >= OUTPUT_EXTRA_DEBUG: import yappi yappi.set_clock_type("cpu") yappi.start() self.log("starting profiler", OUTPUT_ERROR) elif keycode[1] == "f10" and self.debug_level >= OUTPUT_EXTRA_DEBUG: import time import yappi stats = yappi.get_func_stats() filename = f"callgrind.{int(time.time())}.prof" stats.save(filename, type="callgrind") self.log(f"wrote profiling results to {filename}", OUTPUT_ERROR)
def main(argv): # setup the argument parser arg_parser = argparse.ArgumentParser(description='backtest') arg_parser.add_argument('-m', '--mode', required=False, default='B', help="set backtest mode(B or T)") arg_parser.add_argument('-d', '--days', required=False, default=1, type=int, help="set backtest days") arg_parser.add_argument('-sd', '--startDate', required=False, default='', help="set backtest days") arg_parser.add_argument('-s', '--vt_symbol', required=False, default='rb1801', help="set backtest vt_symbol") arg_parser.add_argument('-s2', '--vt_symbol2', required=False, default='', help="set spread vt_symbol2") arg_parser.add_argument('-hd', '--historyDays', required=False, default=0, type=int, help="set history days") arg_parser.add_argument('-sf', '--settingFile', required=False, default='CTA_setting_multi.json', help="setting file name") arg_parser.add_argument('-o', '--optimization', required=False, default=False, type=bool, help="parameter optimization") arg_parser.add_argument('-yappi', '--yappi', required=False, default=False, type=bool, help="yappi status") # parse arguments cmd = arg_parser.parse_args(argv) if cmd.yappi: import yappi yappi.set_clock_type("cpu") yappi.start() backtesting(settingFile=cmd.settingFile, startDate=cmd.startDate, days=cmd.days, mode=cmd.mode, vt_symbol=cmd.vt_symbol, vt_symbol2=cmd.vt_symbol2, historyDays=cmd.historyDays, optimization=cmd.optimization) if cmd.yappi: yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all()
def test_merge_aabab_aabbc(self): _timings = { "a_1": 15, "a_2": 14, "b_1": 12, "a_3": 10, "b_2": 9, "c_1": 4 } _yappi._set_test_timings(_timings) def a(): if self._ncall == 1: self._ncall += 1 a() elif self._ncall == 5: self._ncall += 1 a() else: b() def b(): if self._ncall == 2: self._ncall += 1 a() elif self._ncall == 6: self._ncall += 1 b() elif self._ncall == 7: c() else: return def c(): pass self._ncall = 1 stats = utils.run_and_get_func_stats(a, ) stats.save("ystats1.ys") yappi.clear_stats() _yappi._set_test_timings(_timings) #stats.print_all() self._ncall = 5 stats = utils.run_and_get_func_stats(a, ) stats.save("ystats2.ys") #stats.print_all() def a(): # same name but another function(code object) pass yappi.start() a() stats = yappi.get_func_stats().add(["ystats1.ys", "ystats2.ys"]) #stats.print_all() self.assertEqual(len(stats), 4) fsa = None for stat in stats: if stat.name == "a" and stat.ttot == 45: fsa = stat break self.assertTrue(fsa is not None) self.assertEqual(fsa.ncall, 7) self.assertEqual(fsa.nactualcall, 3) self.assertEqual(fsa.ttot, 45) self.assertEqual(fsa.tsub, 10) fsb = utils.find_stat_by_name(stats, "b") fsc = utils.find_stat_by_name(stats, "c") self.assertEqual(fsb.ncall, 6) self.assertEqual(fsb.nactualcall, 3) self.assertEqual(fsb.ttot, 36) self.assertEqual(fsb.tsub, 27) self.assertEqual(fsb.tavg, 6) self.assertEqual(fsc.ttot, 8) self.assertEqual(fsc.tsub, 8) self.assertEqual(fsc.tavg, 4) self.assertEqual(fsc.nactualcall, fsc.ncall, 2)
def test_subsequent_profile(self): WORKER_COUNT = 5 def a(): pass def b(): pass def c(): pass _timings = { "a_1": 3, "b_1": 2, "c_1": 1, } yappi.start() def g(): pass g() yappi.stop() yappi.clear_stats() _yappi._set_test_timings(_timings) yappi.start() _dummy = [] for i in range(WORKER_COUNT): t = threading.Thread(target=a) t.start() t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=b) t.start() _dummy.append(t) t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=a) t.start() t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=c) t.start() t.join() yappi.stop() yappi.start() def f(): pass f() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'a') fsb = utils.find_stat_by_name(stats, 'b') fsc = utils.find_stat_by_name(stats, 'c') self.assertEqual(fsa.ncall, 10) self.assertEqual(fsb.ncall, 5) self.assertEqual(fsc.ncall, 5) self.assertEqual(fsa.ttot, fsa.tsub, 30) self.assertEqual(fsb.ttot, fsb.tsub, 10) self.assertEqual(fsc.ttot, fsc.tsub, 5) # MACOSx optimizes by only creating one worker thread self.assertTrue(len(yappi.get_thread_stats()) >= 2)
def main(args): do_prof = args.do_profile do_tensorflow = not args.disable_tensorflow #use the environment variable for cupy/cuda choice args.use_cuda = USE_CUPY # Optionally disable pinned memory (will be somewhat slower) if args.use_cuda: import cupy if not args.pinned: cupy.cuda.set_allocator(None) cupy.cuda.set_pinned_memory_allocator(None) datasets = yaml.load(open(args.datasets_yaml), Loader=yaml.FullLoader)["datasets"] #Filter datasets by era datasets_to_process = [] for ds in datasets: if args.datasets is None or ds["name"] in args.datasets: if args.eras is None or ds["era"] in args.eras: datasets_to_process += [ds] if len(datasets_to_process) == 0: raise Exception("No datasets considered, please check the --datasets and --eras options") datasets = datasets_to_process hmumu_utils.NUMPY_LIB, hmumu_utils.ha = choose_backend(args.use_cuda) Dataset.numpy_lib = hmumu_utils.NUMPY_LIB NUMPY_LIB = hmumu_utils.NUMPY_LIB outpath_partial = "{0}/partial_results".format(args.out) try: os.makedirs(outpath_partial) except FileExistsError as e: print("Output path {0} already exists, not recreating".format(outpath_partial)) #save the parameters as a pkl file from pars import analysis_parameters for analysis_name in analysis_parameters.keys(): analysis_parameters[analysis_name]["do_factorized_jec"] = args.do_factorized_jec analysis_parameters[analysis_name]["dnn_vars_path"] = "{0}/dnn_vars".format(args.out) with open('{0}/parameters.pkl'.format(outpath_partial), 'wb') as handle: pickle.dump(analysis_parameters, handle, protocol=pickle.HIGHEST_PROTOCOL) #Recreate dump of all filenames cache_filename = "{0}/datasets.json".format(args.out) use_skim = False if args.cachepath is None: print("--cachepath not specified, will process unskimmed NanoAOD, which is somewhat slower!") print("Please see the README.md on how to skim the NanoAOD") datapath = args.datapath else: print("Processing skimmed NanoAOD") datapath = args.cachepath use_skim = True check_and_recreate_filename_cache(cache_filename, datapath, datasets, use_skim) #Create the jobfiles if args.jobfiles is None: create_all_jobfiles(datasets, cache_filename, datapath, args.chunksize, args.out) #For each dataset, find out which chunks we want to process if "analyze" in args.action: jobfile_data = load_jobfiles(datasets, args.jobfiles_load, args.jobfiles, args.maxchunks, args.out) #Start the profiler only in the actual data processing if do_prof: import yappi yappi.set_clock_type('cpu') yappi.start(builtins=True) #Run the physics analysis on all specified jobfiles if "analyze" in args.action: print("Running the 'analyze' step of the analysis, processing the events into histograms with all systematics") analysis_corrections = AnalysisCorrections(args, do_tensorflow) run_analysis(args, outpath_partial, jobfile_data, analysis_parameters, analysis_corrections) if do_prof: stats = yappi.get_func_stats() stats.save("analysis.prof", type='callgrind') #Merge the partial results (pieces of each dataset) if "merge" in args.action: with ProcessPoolExecutor(max_workers=args.nthreads) as executor: for dataset in datasets: dataset_name = dataset["name"] dataset_era = dataset["era"] is_mc = dataset["is_mc"] fut = executor.submit(merge_partial_results, dataset_name, dataset_era, args.out, outpath_partial) print("done merging") #print memory usage total_memory = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss total_memory += resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print("maxrss={0} MB".format(total_memory/1024))
import yappi #yappi.set_clock_type("waLL") #yappi.start(builtins=True) yappi.start() class A: def bar(self): pass def foo(): def inner_foo(): pass import time time.sleep(2.0) for i in range(20000000): pass a = A() a.bar() inner_foo() foo() #yappi.write_callgrind_stats() yappi.get_func_stats().sort("totaLTiMe").print_all() yappi.get_thread_stats().print_all() #import cProfile #cProfile.run('foo()', 'fooprof') #import pstats #p = pstats.Stats('fooprof') #p.strip_dirs().sort_stats(-1).print_stats()
def profile(config_location="/etc/redeem"): import yappi yappi.start() main(config_location) yappi.get_func_stats().print_all()
def profile(): import yappi yappi.start() main() yappi.get_func_stats().print_all()
def test_merge_stats(self): _timings = { "a_1": 15, "b_1": 14, "c_1": 12, "d_1": 10, "e_1": 9, "f_1": 7, "g_1": 6, "h_1": 5, "i_1": 1 } _yappi._set_test_timings(_timings) def a(): b() def b(): c() def c(): d() def d(): e() def e(): f() def f(): g() def g(): h() def h(): i() def i(): pass yappi.start() a() a() yappi.stop() stats = yappi.get_func_stats() self.assertRaises(NotImplementedError, stats.save, "", "INVALID_SAVE_TYPE") stats.save("ystats2.ys") yappi.clear_stats() _yappi._set_test_timings(_timings) yappi.start() a() stats = yappi.get_func_stats().add("ystats2.ys") fsa = utils.find_stat_by_name(stats, "a") fsb = utils.find_stat_by_name(stats, "b") fsc = utils.find_stat_by_name(stats, "c") fsd = utils.find_stat_by_name(stats, "d") fse = utils.find_stat_by_name(stats, "e") fsf = utils.find_stat_by_name(stats, "f") fsg = utils.find_stat_by_name(stats, "g") fsh = utils.find_stat_by_name(stats, "h") fsi = utils.find_stat_by_name(stats, "i") self.assertEqual(fsa.ttot, 45) self.assertEqual(fsa.ncall, 3) self.assertEqual(fsa.nactualcall, 3) self.assertEqual(fsa.tsub, 3) self.assertEqual(fsa.children[fsb].ttot, fsb.ttot) self.assertEqual(fsa.children[fsb].tsub, fsb.tsub) self.assertEqual(fsb.children[fsc].ttot, fsc.ttot) self.assertEqual(fsb.children[fsc].tsub, fsc.tsub) self.assertEqual(fsc.tsub, 6) self.assertEqual(fsc.children[fsd].ttot, fsd.ttot) self.assertEqual(fsc.children[fsd].tsub, fsd.tsub) self.assertEqual(fsd.children[fse].ttot, fse.ttot) self.assertEqual(fsd.children[fse].tsub, fse.tsub) self.assertEqual(fse.children[fsf].ttot, fsf.ttot) self.assertEqual(fse.children[fsf].tsub, fsf.tsub) self.assertEqual(fsf.children[fsg].ttot, fsg.ttot) self.assertEqual(fsf.children[fsg].tsub, fsg.tsub) self.assertEqual(fsg.ttot, 18) self.assertEqual(fsg.tsub, 3) self.assertEqual(fsg.children[fsh].ttot, fsh.ttot) self.assertEqual(fsg.children[fsh].tsub, fsh.tsub) self.assertEqual(fsh.ttot, 15) self.assertEqual(fsh.tsub, 12) self.assertEqual(fsh.tavg, 5) self.assertEqual(fsh.children[fsi].ttot, fsi.ttot) self.assertEqual(fsh.children[fsi].tsub, fsi.tsub)
def test_profile_decorator(self): def aggregate(func, stats): fname = "%s.profile" % (func.__name__) try: stats.add(fname) except IOError: pass stats.save(fname) raise Exception("messing around") @yappi.profile(return_callback=aggregate) def a(x, y): if x + y == 25: raise Exception("") return x + y def b(): pass try: os.remove( "a.profile") # remove the one from prev test, if available except: pass # global profile is on to mess things up yappi.start() b() # assert functionality and call function at same time try: self.assertEqual(a(1, 2), 3) except: pass try: self.assertEqual(a(2, 5), 7) except: pass try: a(4, 21) except: pass stats = yappi.get_func_stats().add("a.profile") fsa = utils.find_stat_by_name(stats, 'a') self.assertEqual(fsa.ncall, 3) self.assertEqual(len(stats), 1) # b() should be cleared out. @yappi.profile(return_callback=aggregate) def count_down_rec(n): if n == 0: return count_down_rec(n - 1) try: os.remove("count_down_rec.profile" ) # remove the one from prev test, if available except: pass try: count_down_rec(4) except: pass try: count_down_rec(3) except: pass stats = yappi.YFuncStats("count_down_rec.profile") fsrec = utils.find_stat_by_name(stats, 'count_down_rec') self.assertEqual(fsrec.ncall, 9) self.assertEqual(fsrec.nactualcall, 2)
serve(app, host='0.0.0.0', port=6543) if __name__ == '__main__': if len(sys.argv) > 1 and sys.argv[1] == '--profile': try: import yappi except ImportError: print 'You need to install the yappi python module to profile Smewt!' sys.exit(1) try: yappi.start(True) main() finally: yappi.stop() #yappi.get_func_stats().print_all() print '\n\nTHREAD STATS' yappi.get_thread_stats().print_all() filename = 'callgrind.out.%d' % os.getpid() yappi.get_func_stats().save(filename, type='callgrind') print '\nWrote callgrind output to %s' % filename else: main()
and y.split('.')[-1] in data else "" for y in fields ] elif args.show_types: # do nothing pass elif args.verbose and istlog: mavutil.dump_message_verbose(sys.stdout, m) print("") else: # Otherwise we output in a standard Python dict-style format s = "%s.%02u: %s" % (time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timestamp)), int(timestamp * 100.0) % 100, m) if args.show_source: s += " srcSystem=%u srcComponent=%u" % (m.get_srcSystem(), m.get_srcComponent()) if args.show_seq: s += " seq=%u" % m.get_seq() print(s) # Update our last timestamp value. last_timestamp = timestamp if args.show_types: for msgType in available_types: print(msgType) if args.profile: yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all()
return x + y import yappi if __name__ == "__main__": # yappi.set_clock_type("wall") yappi.set_clock_type("cpu") yappi.start() for i in range(10): t = threading.Thread(target=sum, args=( 100, i, ), name="hello" + str(i)) t.start() main_thread = threading.currentThread() for t in threading.enumerate(): if t is not main_thread: t.join() yappi.get_func_stats().print_all( columns={ 0: ("name", 50), 1: ("ncall", 15), 2: ("tsub", 18), 3: ("ttot", 18), 4: ("tavg", 18) }) yappi.get_thread_stats().print_all()