def test_basic(self): yappi.set_clock_type('wall') def dummy(): pass def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') dummy() # call dummy to force ctx name to be retrieved again. self.assertTrue(tsa is not None) # TODO: I put dummy() to fix below, remove the comments after a while. self.assertTrue( # FIX: I see this fails sometimes? tsm is not None, 'Could not find "_MainThread". Found: %s' % (', '.join(utils.get_stat_names(tstats))))
def main() -> None: '''Runs server''' # Parse options define('production', default = False, help = 'run in production mode', type = bool) options.parse_command_line() # Set server name pname = settings.process_name if settings.process_name else None if pname: setproctitle(pname) # Register IRC server server = IRCServer(settings = ircdsettings) for address, port in ircdsettings['listen']: server.listen(port, address = address) # Start profiling if settings.profiling: import yappi yappi.start() # Setup autoreload autoreload.start() # Run application IOLoop.instance().start()
def main4(): import yappi yappi.start() main2() stats = yappi.get_func_stats() stats.sort('ttot') stats.print_all()
def __init__(self, port, num_workers): self._workers = {} self.num_workers = num_workers self._port = port self._server = rpc.listen("0.0.0.0", port) self._server.register_object(self) self._initialized = False self._server.serve_nonblock() self._ctx = None self._worker_statuses = {} self._worker_scores = {} self._available_workers = [] self._arrays = weakref.WeakSet() if FLAGS.profile_master: import yappi yappi.start() atexit.register(_dump_profile) atexit.register(_shutdown) global MASTER MASTER = self
def test_filter(self): def a(): pass def b(): a() def c(): b() _TCOUNT = 5 ts = [] yappi.start() for i in range(_TCOUNT): t = threading.Thread(target=c) t.start() ts.append(t) for t in ts: t.join() yappi.stop() fstats = yappi.get_func_stats(filter={"ctx_id":9}) self.assertTrue(fstats.empty()) fstats = yappi.get_func_stats(filter={"ctx_id":0, "name":"c"}) # main thread self.assertTrue(fstats.empty()) for i in range(1, _TCOUNT): fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"a", "ncall":1}) self.assertEqual(fstats.pop().ncall, 1) fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"b"}) self.assertEqual(fstats.pop().ncall, 1) fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"c"}) self.assertEqual(fstats.pop().ncall, 1)
def test_yappi_overhead(self): import time LOOP_COUNT = 10000 def a(): pass def b(): for i in range(LOOP_COUNT): a() t0 = time.time() yappi.start() b() yappi.stop() time_with_yappi = time.time() - t0 t0 = time.time() b() time_without_yappi = time.time() - t0 if time_without_yappi == 0: time_without_yappi = 0.000001 # in latest v0.82, I calculated this as close to "7.0" in my machine. # however, %83 of this overhead is coming from tickcount(). The other %17 # seems to have been evenly distributed to the internal bookkeeping # structures/algorithms which seems acceptable. Note that our test only # tests one function being profiled at-a-time in a short interval. # profiling high number of functions in a small time # is a different beast, (which is pretty unlikely in most applications) # So as a conclusion: I cannot see any optimization window for Yappi that # is worth implementing as we will only optimize %17 of the time. sys.stderr.write("\r\nYappi puts %0.1f times overhead to the profiled application in average.\r\n" % \ (time_with_yappi / time_without_yappi))
def test_pause_resume(self): yappi.set_context_id_callback(lambda: self.context_id) yappi.set_clock_type('wall') # Start in context 0. self.context_id = 0 yappi.start() time.sleep(0.08) # Switch to context 1. self.context_id = 1 time.sleep(0.05) # Switch back to context 0. self.context_id = 0 time.sleep(0.07) yappi.stop() t_stats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(2, len(t_stats)) self.assertEqual(0, t_stats[0].id) self.assertEqual(2, t_stats[0].sched_count) self.assertTrue(0.15 < t_stats[0].ttot < 0.3) self.assertEqual(1, t_stats[1].id) self.assertEqual(1, t_stats[1].sched_count) # Context 1 was first scheduled 0.08 sec after context 0. self.assertTrue(0.1 < t_stats[1].ttot < 0.2 )
def test_callback(self): self.context_id = 0 yappi.set_context_id_callback(lambda: self.context_id) yappi.start() a() self.context_id = 1 a() self.context_id = 2 a() # Re-schedule context 1. self.context_id = 1 a() yappi.stop() threadstats = yappi.get_thread_stats().sort('id', 'ascending') self.assertEqual(3, len(threadstats)) self.assertEqual(0, threadstats[0].id) self.assertEqual(1, threadstats[1].id) self.assertEqual(2, threadstats[2].id) self.assertEqual(1, threadstats[0].sched_count) self.assertEqual(2, threadstats[1].sched_count) # Context 1 ran twice. self.assertEqual(1, threadstats[2].sched_count) funcstats = yappi.get_func_stats() self.assertEqual(4, utils.find_stat_by_name(funcstats, 'a').ncall)
def test_profile_single_context(self): def id_callback(): return self.callback_count def a(): pass self.callback_count = 1 yappi.set_context_id_callback(id_callback) yappi.start(profile_threads=False) a() # context-id:1 self.callback_count = 2 a() # context-id:2 stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, "a") self.assertEqual(fsa.ncall, 1) yappi.stop() yappi.clear_stats() self.callback_count = 1 yappi.start() # profile_threads=True a() # context-id:1 self.callback_count = 2 a() # context-id:2 stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, "a") self.assertEqual(fsa.ncall, 2)
def test_print_formatting(self): def a(): pass def b(): a() func_cols = {1: ("name", 48), 0: ("ncall", 5), 2: ("tsub", 8)} thread_cols = {1: ("name", 48), 0: ("ttot", 8)} yappi.start() a() b() yappi.stop() fs = yappi.get_func_stats() cs = fs[1].children ts = yappi.get_thread_stats() # fs.print_all(out=sys.stderr, columns={1:("name", 70), }) # cs.print_all(out=sys.stderr, columns=func_cols) # ts.print_all(out=sys.stderr, columns=thread_cols) # cs.print_all(out=sys.stderr, columns={}) self.assertRaises(yappi.YappiError, fs.print_all, columns={1: ("namee", 9)}) self.assertRaises(yappi.YappiError, cs.print_all, columns={1: ("dd", 0)}) self.assertRaises(yappi.YappiError, ts.print_all, columns={1: ("tidd", 0)})
def test_singlethread_profiling(self): yappi.set_clock_type("wall") def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(profile_threads=False) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, "Worker1.a") fsa2 = utils.find_stat_by_name(stats, "a") self.assertTrue(fsa1 is None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa2.ttot > 0.1)
def test_merge_multithreaded_stats(self): import threading import _yappi timings = {"a_1":2, "b_1":1} _yappi._set_test_timings(timings) def a(): pass def b(): pass yappi.start() t = threading.Thread(target=a) t.start() t.join() t = threading.Thread(target=b) t.start() t.join() yappi.get_func_stats().save("ystats1.ys") yappi.clear_stats() _yappi._set_test_timings(timings) self.assertEqual(len(yappi.get_func_stats()), 0) self.assertEqual(len(yappi.get_thread_stats()), 1) t = threading.Thread(target=a) t.start() t.join() self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0) self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1) yappi.get_func_stats().save("ystats2.ys") stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys",]) fsa = utils.find_stat_by_name(stats, "a") fsb = utils.find_stat_by_name(stats, "b") self.assertEqual(fsa.ncall, 2) self.assertEqual(fsb.ncall, 1) self.assertEqual(fsa.tsub, fsa.ttot, 4) self.assertEqual(fsb.tsub, fsb.ttot, 1)
def test_children_stat_functions(self): _timings = {"a_1":5, "b_1":3, "c_1":1} _yappi._set_test_timings(_timings) def b(): pass def c(): pass def a(): b() c() yappi.start() a() b() # non-child call c() # non-child call stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'a') childs_of_a = fsa.children.get().sort("tavg", "desc") prev_item = None for item in childs_of_a: if prev_item: self.assertTrue(prev_item.tavg > item.tavg) prev_item = item childs_of_a.sort("name", "desc") prev_item = None for item in childs_of_a: if prev_item: self.assertTrue(prev_item.name > item.name) prev_item = item childs_of_a.clear() self.assertTrue(childs_of_a.empty())
def test_basic(self): import threading import time yappi.set_clock_type('wall') def a(): time.sleep(0.2) class Worker1(threading.Thread): def a(self): time.sleep(0.3) def run(self): self.a() yappi.start(builtins=False, profile_threads=True) c = Worker1() c.start() c.join() a() stats = yappi.get_func_stats() fsa1 = utils.find_stat_by_name(stats, 'Worker1.a') fsa2 = utils.find_stat_by_name(stats, 'a') self.assertTrue(fsa1 is not None) self.assertTrue(fsa2 is not None) self.assertTrue(fsa1.ttot > 0.2) self.assertTrue(fsa2.ttot > 0.1) tstats = yappi.get_thread_stats() self.assertEqual(len(tstats), 2) tsa = utils.find_stat_by_name(tstats, 'Worker1') tsm = utils.find_stat_by_name(tstats, '_MainThread') self.assertTrue(tsa is not None) self.assertTrue(tsm is not None) # FIX: I see this fails sometimes?
def start(self): yappi.set_context_id_callback(lambda:id(greenlet.getcurrent())) yappi.set_context_name_callback(lambda:greenlet.getcurrent().__class__.__name__) yappi.set_clock_type('cpu') yappi.start(builtins=True) self._isStart = True return 'success'
def test_producer_consumer_with_queues(self): # we currently just stress yappi, no functionality test is done here. yappi.start() import time if utils.is_py3x(): from queue import Queue else: from Queue import Queue from threading import Thread WORKER_THREAD_COUNT = 50 WORK_ITEM_COUNT = 2000 def worker(): while True: item = q.get() # do the work with item q.task_done() q = Queue() for i in range(WORKER_THREAD_COUNT): t = Thread(target=worker) t.daemon = True t.start() for item in range(WORK_ITEM_COUNT): q.put(item) q.join()# block until all tasks are done #yappi.get_func_stats().sort("callcount").print_all() yappi.stop()
def main(): ap = argparse.ArgumentParser() ap.add_argument('--host', default=None, # NOT -h, that's help help='host that coordinated will listen on, ' '0.0.0.0 for any input interface') ap.add_argument('--port', '-p', type=int, default=None, help='port number that coordinated will listen on') ap.add_argument('--pid', default=None, help='file to write pid to') ap.add_argument('--snapshot-dir', default=None, help='direcotry to write snapshots to') ap.add_argument('--httpd', default=None, help='ip:port or :port to serve http info on') if yappi is not None: ap.add_argument('--yappi', default=None, help='file to write yappi profiling to. will be suffied by {timestamp}.txt') args = yakonfig.parse_args(ap, [yakonfig, dblogger, coordinate]) if args.pid: with open(args.pid, 'w') as f: f.write(str(os.getpid())) if args.snapshot_dir is not None: cjqconfig = yakonfig.get_global_config('coordinate', 'job_queue') # (This modifies the global configuration in place) cjqconfig['snapshot_path_format'] = os.path.join( args.snapshot_dir, 'snapshot_{timestamp}') if (yappi is not None) and args.yappi: yappi.start() yt = threading.Thread(target=yappi_logger, args=(args.yappi,)) yt.daemon = True yt.start() daemon = CoordinateServer(host=args.host, port=args.port, httpd=args.httpd) daemon.run()
def test_concurrent_futures(self): yappi.start() from concurrent.futures import ThreadPoolExecutor with ThreadPoolExecutor(max_workers=5) as executor: f = executor.submit(pow, 5, 2) self.assertEqual(f.result(), 25) time.sleep(1.0) yappi.stop()
def wrapped(*args, **kwargs): yappi.start() result = func(*args, **kwargs) yappi.stop() prof_file = "%s.%s" % (func.__name__, time.time()) #prof_file = "callgrind.a.1" yappi.get_func_stats().save(prof_file, "ystat") return result
def test_lambda(self): f = lambda: time.sleep(0.3) yappi.set_clock_type("wall") yappi.start() f() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, "<lambda>") self.assertTrue(fsa.ttot > 0.1)
def get(self): self.interval = int(self.get_argument('interval', self.interval)) self.sort_type = yappi.SORT_TYPES_FUNCSTATS[self.get_argument( 'sort_type', 'tsub')] self.limit = self.get_argument('limit', self.limit) yappi.start() tornado.ioloop.IOLoop.instance().add_timeout( datetime.timedelta(seconds=self.interval), self.finish_profile)
def test_start_flags(self): self.assertEqual(_yappi._get_start_flags(), None) yappi.start() def a(): pass a() self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0) self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1) self.assertEqual(len(yappi.get_thread_stats()), 1)
def __init__(self, master): # Reseed the Numpy random number state. # # The default initialization results in all worker processes on the machine having the # same random seed. np.random.seed(seed=os.getpid()) self.id = -1 self._initialized = False self._peers = {} self._blobs = {} self._master = master self._running = True self._ctx = None self.worker_status = core.WorkerStatus( psutil.TOTAL_PHYMEM, psutil.NUM_CPUS, psutil.virtual_memory().percent, psutil.cpu_percent(), time.time(), [], [], ) self._lock = rlock.FastRLock() # self._lock = threading.Lock() # Patch to fix buggy assumption by multiprocessing library if not hasattr(threading.current_thread(), "_children"): threading.current_thread()._children = weakref.WeakKeyDictionary() self._kernel_threads = ThreadPool(processes=1) self._kernel_remain_tiles = [] if FLAGS.profile_worker: import yappi yappi.start() hostname = socket.gethostname() self._server = rpc.listen_on_random_port(hostname) self._server.register_object(self) if FLAGS.profile_worker: self._server._socket._event_loop.enable_profiling() self._server.serve_nonblock() req = core.RegisterReq() req.host = hostname req.port = self._server.addr[1] req.worker_status = self.worker_status with _init_lock: # There is a race-condition in the initialization code for zeromq; this causes # sporadic crashes when running in multi-thread mode. We lock the first # client RPC to workaround this issue. master.register(req)
def test_generator(self): def _gen(n): while(n > 0): yield n n -= 1 yappi.start() for x in _gen(5): pass self.assertTrue(yappi.convert2pstats(yappi.get_func_stats()) is not None)
def start(): pages.require("/admin/settings.edit", noautoreturn=True) pages.postOnly() import yappi if not yappi.is_running(): yappi.start() time.sleep(0.5) messagebus.postMessage("/system/settings/activatedprofiler",pages.getAcessingUser()) raise cherrypy.HTTPRedirect("/settings/profiler")
def _start_profiling(self, clock): with lock: if yappi.is_running(): raise http.Error( http.BAD_REQUEST, "profile is already running") log.info("Starting profiling using %r clock", clock) yappi.set_clock_type(clock) yappi.start(builtins=True, profile_threads=True)
def test_clear_stats_while_running(self): def a(): pass yappi.start() a() yappi.clear_stats() a() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'a') self.assertEqual(fsa.ncall, 1)
def yappi_prof_call(func, *args): ''' https://code.google.com/p/yappi/wiki/usageyappi_v092 ''' import yappi yappi.start() result = func(*args) yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() return result
def start_cpu_profiling(): """ If the environment variable W3AF_PROFILING is set to 1, then we start the CPU and memory profiling. :return: None """ import yappi yappi.start() dump_data_every_thread(dump_data, DELAY_MINUTES, SAVE_THREAD_PTR)
def profile(): """ Code profiler using YAPPI http://code.google.com/p/yappi """ import yappi yappi.start() start(False) yappi.stop() for pstat in yappi.get_stats(yappi.SORTTYPE_TSUB): print pstat
def main(args, datasets): do_prof = args.do_profile do_tensorflow = not args.disable_tensorflow #use the environment variable for cupy/cuda choice args.use_cuda = USE_CUPY analysis_corrections = None if "analyze" in args.action: analysis_corrections = AnalysisCorrections(args, do_tensorflow) # Optionally disable pinned memory (will be somewhat slower) if args.use_cuda: import cupy if not args.pinned: cupy.cuda.set_allocator(None) cupy.cuda.set_pinned_memory_allocator(None) #Use sync-only datasets if args.do_sync: datasets = datasets_sync #Filter datasets by era datasets_to_process = [] for ds in datasets: if args.datasets is None or ds[0] in args.datasets: if args.eras is None or ds[1] in args.eras: datasets_to_process += [ds] print("Will consider dataset", ds) if len(datasets) == 0: raise Exception("No datasets considered, please check the --datasets and --eras options") datasets = datasets_to_process hmumu_utils.NUMPY_LIB, hmumu_utils.ha = choose_backend(args.use_cuda) Dataset.numpy_lib = hmumu_utils.NUMPY_LIB NUMPY_LIB = hmumu_utils.NUMPY_LIB # All analysis definitions (cut values etc) should go here analysis_parameters = { "baseline": { "nPV": 0, "NdfPV": 4, "zPV": 24, # Will be applied with OR "hlt_bits": { "2016": ["HLT_IsoMu24", "HLT_IsoTkMu24"], "2017": ["HLT_IsoMu27"], "2018": ["HLT_IsoMu24"], }, "muon_pt": 20, "muon_pt_leading": {"2016": 26.0, "2017": 29.0, "2018": 26.0}, "muon_eta": 2.4, "muon_iso": 0.25, "muon_id": {"2016": "medium", "2017": "medium", "2018": "medium"}, "muon_trigger_match_dr": 0.1, "muon_iso_trigger_matched": 0.15, "muon_id_trigger_matched": {"2016": "tight", "2017": "tight", "2018": "tight"}, "do_rochester_corrections": True, "do_lepton_sf": True, "do_jec": True, "jec_tag": {"2016": "Summer16_07Aug2017_V11", "2017": "Fall17_17Nov2017_V32", "2018": "Autumn18_V16"}, "jet_mu_dr": 0.4, "jet_pt_leading": {"2016": 35.0, "2017": 35.0, "2018": 35.0}, "jet_pt_subleading": {"2016": 25.0, "2017": 25.0, "2018": 25.0}, "jet_eta": 4.7, "jet_id": "tight", "jet_puid": "loose", "jet_veto_eta": [2.65, 3.139], "jet_veto_raw_pt": 50.0, "jet_btag": {"2016": 0.6321, "2017": 0.4941, "2018": 0.4184}, "do_factorized_jec": args.do_factorized_jec, "cat5_dijet_inv_mass": 400.0, "cat5_abs_jj_deta_cut": 2.5, "masswindow_z_peak": [76, 106], "masswindow_h_sideband": [110, 150], "masswindow_h_peak": [115, 135], "inv_mass_bins": 41, "extra_electrons_pt": 20, "extra_electrons_eta": 2.5, "extra_electrons_iso": 0.4, #Check if we want to apply this "extra_electrons_id": "mvaFall17V1Iso_WP90", "save_dnn_vars": True, "dnn_vars_path": "{0}/dnn_vars".format(args.out), #If true, apply mjj > cut, otherwise inverse "vbf_filter_mjj_cut": 350, "vbf_filter": { "dy_m105_160_mg": True, "dy_m105_160_amc": True, "dy_m105_160_vbf_mg": False, "dy_m105_160_vbf_amc": False, }, #Irene's DNN input variable order for keras "dnn_varlist_order": ['softJet5', 'dRmm','dEtamm','M_jj','pt_jj','eta_jj','phi_jj','M_mmjj','eta_mmjj','phi_mmjj','dEta_jj','Zep','dRmin_mj', 'dRmax_mj', 'dRmin_mmj','dRmax_mmj','dPhimm','leadingJet_pt','subleadingJet_pt', 'leadingJet_eta','subleadingJet_eta','leadingJet_qgl','subleadingJet_qgl','cthetaCS','Higgs_pt','Higgs_eta','Higgs_mass'], "dnn_input_histogram_bins": { "softJet5": (0,10,10), "dRmm": (0,5,41), "dEtamm": (-2,2,41), "dPhimm": (-2,2,41), "M_jj": (0,2000,41), "pt_jj": (0,400,41), "eta_jj": (-5,5,41), "phi_jj": (-5,5,41), "M_mmjj": (0,2000,41), "eta_mmjj": (-3,3,41), "phi_mmjj": (-3,3,41), "dEta_jj": (-3,3,41), "Zep": (-2,2,41), "dRmin_mj": (0,5,41), "dRmax_mj": (0,5,41), "dRmin_mmj": (0,5,41), "dRmax_mmj": (0,5,41), "leadingJet_pt": (0, 200, 41), "subleadingJet_pt": (0, 200, 41), "leadingJet_eta": (-5, 5, 41), "subleadingJet_eta": (-5, 5, 41), "leadingJet_qgl": (0, 1, 41), "subleadingJet_qgl": (0, 1, 41), "cthetaCS": (-1, 1, 41), "Higgs_pt": (0, 200, 41), "Higgs_eta": (-3, 3, 41), "Higgs_mass": (110, 150, 41), "dnn_pred": (0, 1, 1001), "dnn_pred2": (0, 1, 11), "bdt_ucsd": (-1, 1, 41), "bdt2j_ucsd": (-1, 1, 41), "bdt01j_ucsd": (-1, 1, 41), "MET_pt": (0, 200, 41), "hmmthetacs": (-1, 1, 41), "hmmphics": (-4, 4, 41), }, "categorization_trees": {} }, } histo_bins = { "muon_pt": np.linspace(0, 200, 101, dtype=np.float32), "npvs": np.linspace(0,100,101, dtype=np.float32), "dijet_inv_mass": np.linspace(0, 2000, 41, dtype=np.float32), "inv_mass": np.linspace(70, 150, 41, dtype=np.float32), "numjet": np.linspace(0, 10, 11, dtype=np.float32), "jet_pt": np.linspace(0, 300, 101, dtype=np.float32), "jet_eta": np.linspace(-4.7, 4.7, 41, dtype=np.float32), "pt_balance": np.linspace(0, 5, 41, dtype=np.float32), "numjets": np.linspace(0, 10, 11, dtype=np.float32), "jet_qgl": np.linspace(0, 1, 41, dtype=np.float32), "higgs_inv_mass_uncertainty": np.linspace(0, 10, 101, dtype=np.float32), "higgs_rel_inv_mass_uncertainty": np.linspace(0, 0.05, 101, dtype=np.float32) } for hname, bins in analysis_parameters["baseline"]["dnn_input_histogram_bins"].items(): histo_bins[hname] = np.linspace(bins[0], bins[1], bins[2], dtype=np.float32) for masswindow in ["z_peak", "h_peak", "h_sideband"]: mw = analysis_parameters["baseline"]["masswindow_" + masswindow] histo_bins["inv_mass_{0}".format(masswindow)] = np.linspace(mw[0], mw[1], 41, dtype=np.float32) histo_bins["dnn_pred2"] = { "h_peak": np.array([0., 0.9, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 1.0], dtype=np.float32), "z_peak": np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 1.0], dtype=np.float32), "h_sideband": np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0], dtype=np.float32), } analysis_parameters["baseline"]["histo_bins"] = histo_bins #analysis_parameters["oldjec"] = copy.deepcopy(analysis_parameters["baseline"]) #analysis_parameters["oldjec"]["jec_tag"]["2018"] = "Autumn18_V8" #Run baseline analysis outpath = "{0}/partial_results".format(args.out) try: os.makedirs(outpath) except FileExistsError as e: pass with open('{0}/parameters.pkl'.format(outpath), 'wb') as handle: pickle.dump(analysis_parameters, handle, protocol=pickle.HIGHEST_PROTOCOL) #Recreate dump of all filenames cache_filename = args.cache_location + "/datasets.json" if ("cache" in args.action) and (args.jobfiles is None): print("--action cache and no jobfiles specified, creating datasets.json dump of all filenames") if not os.path.isdir(args.cache_location): os.makedirs(args.cache_location) filenames_cache = {} for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset filenames_all = glob.glob(args.datapath + dataset_globpattern, recursive=True) filenames_all = [fn for fn in filenames_all if not "Friend" in fn] filenames_cache[dataset_name + "_" + dataset_era] = [ fn.replace(args.datapath, "") for fn in filenames_all] if len(filenames_all) == 0: raise Exception("Dataset {0} matched 0 files from glob pattern {1}, verify that the data files are located in {2}".format( dataset_name, dataset_globpattern, args.datapath )) #save all dataset filenames to a json file print("Creating a json dump of all the dataset filenames based on data found in {0}".format(args.datapath)) if os.path.isfile(cache_filename): print("Cache file {0} already exists, we will not overwrite it to be safe.".format(cache_filename), file=sys.stderr) print("Delete it or change --cache-location and try again.", file=sys.stderr) sys.exit(1) with open(cache_filename, "w") as fi: fi.write(json.dumps(filenames_cache, indent=2)) if ("cache" in args.action or "analyze" in args.action) and (args.jobfiles is None): #Create a list of job files for processing jobfile_data = [] print("Loading list of filenames from {0}".format(cache_filename)) if not os.path.isfile(cache_filename): raise Exception("Cached dataset list of filenames not found in {0}, please run this code with --action cache".format( cache_filename)) filenames_cache = json.load(open(cache_filename, "r")) for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset try: filenames_all = filenames_cache[dataset_name + "_" + dataset_era] except KeyError as e: print("Could not load {0} from {1}, please make sure this dataset has been added to cache".format( dataset_name + "_" + dataset_era, cache_filename), file=sys.stderr) raise e filenames_all_full = [args.datapath + "/" + fn for fn in filenames_all] chunksize = args.chunksize * chunksize_multiplier.get(dataset_name, 1) print("Saving dataset {0}_{1} with {2} files in {3} files per chunk to jobfiles".format( dataset_name, dataset_era, len(filenames_all_full), chunksize)) jobfile_dataset = create_dataset_jobfiles(dataset_name, dataset_era, filenames_all_full, is_mc, chunksize, args.out) jobfile_data += jobfile_dataset print("Dataset {0}_{1} consists of {2} chunks".format( dataset_name, dataset_era, len(jobfile_dataset))) assert(len(jobfile_data) > 0) assert(len(jobfile_data[0]["filenames"]) > 0) #For each dataset, find out which chunks we want to process if "cache" in args.action or "analyze" in args.action: jobfile_data = [] if not (args.jobfiles_load is None): args.jobfiles = [l.strip() for l in open(args.jobfiles_load).readlines()] if args.jobfiles is None: print("You did not specify to process specific dataset chunks, assuming you want to process all chunks") print("If this is not true, please specify e.g. --jobfiles data_2018_0.json data_2018_1.json ...") args.jobfiles = [] for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset jobfiles_dataset = glob.glob(args.out + "/jobfiles/{0}_{1}_*.json".format(dataset_name, dataset_era)) assert(len(jobfiles_dataset) > 0) if args.maxchunks > 0: jobfiles_dataset = jobfiles_dataset[:args.maxchunks] args.jobfiles += jobfiles_dataset #Now load the jobfiles assert(len(args.jobfiles) > 0) print("You specified --jobfiles {0}, processing only these dataset chunks".format(" ".join(args.jobfiles))) jobfile_data = [] for f in args.jobfiles: jobfile_data += [json.load(open(f))] chunkstr = " ".join(["{0}_{1}_{2}".format( ch["dataset_name"], ch["dataset_era"], ch["dataset_num_chunk"]) for ch in jobfile_data]) print("Will process {0} dataset chunks: {1}".format(len(jobfile_data), chunkstr)) assert(len(jobfile_data) > 0) #Start the profiler only in the actual data processing if do_prof: import yappi filename = 'analysis.prof' yappi.set_clock_type('cpu') yappi.start(builtins=True) if "cache" in args.action: print("Running the 'cache' step of the analysis, ROOT files will be opened and branches will be uncompressed") print("Will retrieve dataset filenames based on existing ROOT files on filesystem in datapath={0}".format(args.datapath)) try: os.makedirs(cmdline_args.cache_location) except Exception as e: pass run_cache(args, outpath, jobfile_data, analysis_parameters) if "analyze" in args.action: run_analysis(args, outpath, jobfile_data, analysis_parameters, analysis_corrections) if "merge" in args.action: with ProcessPoolExecutor(max_workers=args.nthreads) as executor: for dataset in datasets: dataset_name, dataset_era, dataset_globpattern, is_mc = dataset fut = executor.submit(merge_partial_results, dataset_name, dataset_era, outpath) print("done merging") if do_prof: stats = yappi.get_func_stats() stats.save(filename, type='callgrind') import resource total_memory = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss total_memory += resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print("maxrss={0} MB".format(total_memory/1024))
def setup_profiler(self): if not self.is_profiling(): return yappi.start()
def injection_load(**kwargs): import yappi if not yappi.is_running(): yappi.start()
def __init__(self): yappi.start() self.timer = threading.Timer(self.TRACING_INTERVAL, self._logTracingStats) self.timer.start()
def main(args): do_prof = args.do_profile do_tensorflow = not args.disable_tensorflow # use the environment variable for cupy/cuda choice args.use_cuda = USE_CUPY datasets = yaml.load(open(args.datasets_yaml), Loader=yaml.FullLoader)["datasets"] # Filter datasets by era datasets_to_process = [] for ds in datasets: if args.datasets is None or ds["name"] in args.datasets: if args.eras is None or ds["era"] in args.eras: datasets_to_process += [ds] if len(datasets_to_process) == 0: raise Exception( "No datasets considered, please check the --datasets and --eras options" ) datasets = datasets_to_process # Choose either the CPU or GPU(CUDA) backend hmumu_utils.NUMPY_LIB, hmumu_utils.ha = choose_backend(args.use_cuda) Dataset.numpy_lib = hmumu_utils.NUMPY_LIB outpath_partial = "{0}/partial_results".format(args.out) try: os.makedirs(outpath_partial) except FileExistsError: print("Output path {0} already exists, not recreating".format( outpath_partial)) # save the parameters as a pkl file from pars import analysis_parameters for analysis_name in analysis_parameters.keys(): analysis_parameters[analysis_name][ "do_factorized_jec"] = args.do_factorized_jec analysis_parameters[analysis_name][ "dnn_vars_path"] = "{0}/dnn_vars".format(args.out) with open('{0}/parameters.pkl'.format(outpath_partial), 'wb') as handle: pickle.dump(analysis_parameters, handle, protocol=pickle.HIGHEST_PROTOCOL) # Recreate dump of all filenames cache_filename = "{0}/datasets.json".format(args.out) use_skim = False if args.cachepath is None: print( "--cachepath not specified, will process unskimmed NanoAOD, which is somewhat slower!" ) print("Please see the README.md on how to skim the NanoAOD") datapath = args.datapath else: print("Processing skimmed NanoAOD") datapath = args.cachepath use_skim = True check_and_recreate_filename_cache(cache_filename, datapath, datasets, use_skim) # Create the jobfiles if args.jobfiles is None: create_all_jobfiles(datasets, cache_filename, datapath, args.chunksize, args.out) # For each dataset, find out which chunks we want to process if "analyze" in args.action: jobfile_data = load_jobfiles(datasets, args.jobfiles_load, args.jobfiles, args.maxchunks, args.out) # If we want to check what part of the code is slow, start the profiler only in the actual data processing if do_prof: import yappi yappi.set_clock_type('cpu') yappi.start(builtins=True) # Run the physics analysis on all specified jobfiles if "analyze" in args.action: print( "Running the 'analyze' step of the analysis, processing the events into histograms with all systematics" ) analysis_corrections = AnalysisCorrections(args, do_tensorflow) run_analysis(args, outpath_partial, jobfile_data, analysis_parameters, analysis_corrections) if do_prof: stats = yappi.get_func_stats() stats.save("analysis.prof", type='callgrind') # Merge the partial results (pieces of each dataset) if "merge" in args.action: with ProcessPoolExecutor(max_workers=args.nthreads) as executor: for dataset in datasets: dataset_name = dataset["name"] dataset_era = dataset["era"] executor.submit(merge_partial_results, dataset_name, dataset_era, args.out, outpath_partial) print("done merging") # print memory usage for debugging total_memory = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss total_memory += resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print("maxrss={0} MB".format(total_memory / 1024))
def run_with_yappi(func, *args, **kwargs): yappi.start() func(*args, **kwargs) yappi.stop()
from plugin_manager import * from enthought.traits.api import HasTraits, Str, Regex, List, Instance, DelegatesTo from enthought.traits.ui.api import TreeEditor, TreeNode, View, Item, VSplit, \ HGroup, Controller, Handler, Group, Include, ValueEditor, HSplit, ListEditor, InstanceEditor from enthought.traits.ui.menu import Menu, Action, Separator, MenuBar import yaml import wx PROFILE = False PROFILE_BUILTINS = True if PROFILE: import yappi yappi.start(PROFILE_BUILTINS) def open_file(wildcard='', dir='./saved_sessions', file_name='', message='Open'): dialog = wx.FileDialog(None, wildcard=wildcard, defaultFile=file_name, defaultDir=dir, message=message, style=wx.OPEN | wx.FILE_MUST_EXIST) file_name = '' if dialog.ShowModal() == wx.ID_OK: file_name = dialog.GetPath()
def setUp(self): self.prof_start = prof.start() yappi.start(builtins=False)
def test_basic(self): @asyncio.coroutine def a(): yield from async_sleep(0.3) burn_cpu(0.4) @asyncio.coroutine def b(): yield from a() @asyncio.coroutine def recursive_a(n): if not n: return burn_io(0.3) yield from async_sleep(0.3) yield from recursive_a(n - 1) tlocal = threading.local() def tag_cbk(): try: return tlocal._tag except: return -1 yappi.set_clock_type("wall") tlocal._tag = 0 yappi.set_tag_callback(tag_cbk) def _thread_event_loop(loop, tag): tlocal._tag = tag asyncio.set_event_loop(loop) loop.run_forever() _TCOUNT = 3 _ctag = 1 ts = [] for i in range(_TCOUNT): _loop = asyncio.new_event_loop() t = threading.Thread(target=_thread_event_loop, args=(_loop, _ctag)) t._loop = _loop t.start() ts.append(t) _ctag += 1 @asyncio.coroutine def stop_loop(): asyncio.get_event_loop().stop() @asyncio.coroutine def driver(): futs = [] fut = asyncio.run_coroutine_threadsafe(a(), ts[0]._loop) futs.append(fut) fut = asyncio.run_coroutine_threadsafe(recursive_a(5), ts[1]._loop) futs.append(fut) fut = asyncio.run_coroutine_threadsafe(b(), ts[2]._loop) futs.append(fut) for fut in futs: fut.result() # stop asyncio loops in threads for t in ts: asyncio.run_coroutine_threadsafe(stop_loop(), t._loop) yappi.start() asyncio.get_event_loop().run_until_complete(driver()) yappi.stop() traces = yappi.get_func_stats() t1 = ''' tests/test_asyncio.py:137 driver 1 0.000061 3.744064 3.744064 tests/test_asyncio.py:96 recursive_a 6/1 0.000188 3.739663 0.623277 tests/test_asyncio.py:8 async_sleep 7 0.000085 2.375271 0.339324 tests/utils.py:135 burn_io 5 0.000044 1.700000 0.437400 tests/test_asyncio.py:87 a 2 0.000019 1.600000 0.921138 tests/utils.py:126 burn_cpu 2 0.800000 0.800000 0.509730 tests/test_asyncio.py:92 b 1 0.000005 0.800000 0.921055 ''' self.assert_traces_almost_equal(t1, traces) traces = yappi.get_func_stats(filter={'tag': 2}) t1 = ''' tests/test_asyncio.py:96 recursive_a 6/1 0.000211 3.720011 0.620002 tests/utils.py:135 burn_io 5 0.000079 1.700000 0.431813 async_sleep 5 0.000170 1.560735 0.312147 ''' self.assert_traces_almost_equal(t1, traces)
def test_profile_decorator(self): def aggregate(func, stats): fname = "%s.profile" % (func.__name__) try: stats.add(fname) except IOError: pass stats.save(fname) raise Exception("messing around") @yappi.profile(return_callback=aggregate) def a(x, y): if x + y == 25: raise Exception("") return x + y def b(): pass try: os.remove( "a.profile") # remove the one from prev test, if available except: pass # global profile is on to mess things up yappi.start() b() # assert functionality and call function at same time try: self.assertEqual(a(1, 2), 3) except: pass try: self.assertEqual(a(2, 5), 7) except: pass try: a(4, 21) except: pass stats = yappi.get_func_stats().add("a.profile") fsa = utils.find_stat_by_name(stats, 'a') self.assertEqual(fsa.ncall, 3) self.assertEqual(len(stats), 1) # b() should be cleared out. @yappi.profile(return_callback=aggregate) def count_down_rec(n): if n == 0: return count_down_rec(n - 1) try: os.remove("count_down_rec.profile" ) # remove the one from prev test, if available except: pass try: count_down_rec(4) except: pass try: count_down_rec(3) except: pass stats = yappi.YFuncStats("count_down_rec.profile") fsrec = utils.find_stat_by_name(stats, 'count_down_rec') self.assertEqual(fsrec.ncall, 9) self.assertEqual(fsrec.nactualcall, 2)
if opts.aircraft is not None: start_script = os.path.join(opts.aircraft, "mavinit.scr") if os.path.exists(start_script): run_script(start_script) else: print("no script %s" % start_script) if opts.cmd is not None: for cstr in opts.cmd: cmds = cstr.split(';') for c in cmds: process_stdin(c) if opts.profile: import yappi # We do the import here so that we won't barf if run normally and yappi not available yappi.start() # log all packets from the master, for later replay open_telemetry_logs(logpath_telem, logpath_telem_raw) # run main loop as a thread mpstate.status.thread = threading.Thread(target=main_loop, name='main_loop') mpstate.status.thread.daemon = True mpstate.status.thread.start() # use main program for input. This ensures the terminal cleans # up on exit while (mpstate.status.exit != True): try: if opts.daemon: time.sleep(0.1)
async def profile_tasks_coroutine(self): yappi.start() tasks = self.get_tasks() await asyncio.gather(*tasks) yappi.stop()
def start(self): yappi.start()
import yappi def estat(e): print e def foo(): pass yappi.start(True) for i in xrange(100000): foo() yappi.stop() #yappi.enum_stats(estat) yappi.print_stats() yappi.clear_stats()
import yappi import threading class deadlock_thread(threading.Thread): # acquire the lock two times to create a very # simple deadlock:) def foo(self): for i in xrange(0, 2): self.lock.acquire() def run(self): self.lock = threading.Lock() self.foo() yappi.start() # start the profiler thr = deadlock_thread() # start the deadlock thread thr.start() while 1: s = raw_input("yappi>") exec s
def enable(self): if self.wall: yappi.clear_stats() yappi.set_clock_type("wall") yappi.start()
def main(args): """Main entry point allowing external calls Args: args ([str]): command line parameter list """ # Create $HOME/tmp directory if it does not exist (Path.home() / "tmp").mkdir(exist_ok=True) # Define logger format and handlers logger = logging.getLogger("transfer_vn") # create file handler which logs even debug messages fh = TimedRotatingFileHandler( str(Path.home()) + "/tmp/transfer_vn.log", when="midnight", interval=1, backupCount=100, ) # create console handler with a higher log level ch = logging.StreamHandler() # create formatter and add it to the handlers formatter = logging.Formatter( "%(asctime)s - %(levelname)s - %(name)s - %(message)s") fh.setFormatter(formatter) ch.setFormatter(formatter) # add the handlers to the logger logger.addHandler(fh) logger.addHandler(ch) # Get command line arguments args = arguments(args) # Start profiling if required if args.profile: yappi.start() logger.info(_("Started yappi")) # Define verbosity if args.verbose: logger.setLevel(logging.DEBUG) sql_quiet = "" client_min_message = "debug1" else: logger.setLevel(logging.INFO) sql_quiet = "--quiet" client_min_message = "warning" logger.info(_("%s, version %s"), sys.argv[0], __version__) logger.info(_("Arguments: %s"), sys.argv[1:]) # If required, first create YAML file if args.init: logger.info(_("Creating YAML configuration file")) init(args.file) return None # Get configuration from file if not (Path.home() / args.file).is_file(): logger.critical(_("File %s does not exist"), str(Path.home() / args.file)) return None logger.info(_("Getting configuration data from %s"), args.file) try: cfg_ctrl = EvnConf(args.file) except YAMLValidationError: logger.critical(_("Incorrect content in YAML configuration %s"), args.file) sys.exit(0) cfg_site_list = cfg_ctrl.site_list cfg = list(cfg_site_list.values())[0] # Check configuration consistency if cfg.db_enabled and cfg.json_format != "short": logger.critical( _("Storing to Postgresql cannot use long json_format.")) logger.critical(_("Please modify YAML configuration and restart.")) sys.exit(0) manage_pg = PostgresqlUtils(cfg) if args.db_drop: logger.info(_("Delete if exists database and roles")) manage_pg.drop_database() if args.db_create: logger.info(_("Create database and roles")) manage_pg.create_database() if args.json_tables_create: logger.info(_("Create, if not exists, json tables")) manage_pg.create_json_tables() if args.col_tables_create: logger.info(_("Creating or recreating vn columns based tables")) col_table_create(cfg, sql_quiet, client_min_message) if args.full: logger.info(_("Performing a full download")) full_download(cfg_ctrl) if args.schedule: logger.info(_("Creating or modifying incremental download schedule")) increment_schedule(cfg_ctrl) if args.update: logger.info(_("Performing an incremental download")) increment_download(cfg_ctrl) if args.status: logger.info(_("Printing download status")) status(cfg_ctrl) if args.count: logger.info(_("Counting observations")) count_observations(cfg_ctrl) # Stop and output profiling if required if args.profile: logger.info(_("Printing yappi results")) yappi.stop() yappi.get_func_stats().print_all() yappi.get_thread_stats().print_all() return None
def clusterArgInicial(idtese): #Variaveis e funçoes para conexação com o banco de dados do Debate de Teses cursor = connection.cursor() cursor2 = connection.cursor() cursor.execute( "select distinct `usr`.`primeironome` as `name`, `arg`.`argumento` AS `posicionamentoinicial` from ((((`argumento` `arg` join `revisao` `rev`) join `replica` `rep`) join `posicionamento` `pos`) join `argumentador` `urg`)join `usuario` `usr` where ((`arg`.`tese_idtese` = " + idtese + " ) and (`rev`.`argumento_idargumento` = `arg`.`idargumento`) and (`rep`.`revisao_idrevisao` = `rev`.`idrevisao`) and (`arg`.`argumentador_idargumentador` = `pos`.`argumentador_idargumentador`) and (`arg`.`tese_idtese` = `pos`.`tese_idtese`) and (`arg`.`posicionamentoinicial` is not null) and (`arg`.`argumentador_idargumentador` = `urg`.`idargumentador`) and(`urg`.`usuario_idusuario` = `usr`.`idusuario`) and (`pos`.`posicionamentofinal` is not null))" ) cursor2.execute("select tese from tese where idtese=" + idtese) #Variavel e função para tratar tags html e acentos com codificação ISO h = HTMLParser.HTMLParser() #dados retirados da consulta ao banco dadosSql = cursor.fetchall() textotese = cursor2.fetchall() #listas para tratar os dados iniciais usu = [] posInicial = [] dados = [] tese = [] #lista com dados pos tagger tag_posInicial = [] tag_comAce_posInicial = [] #lista com dados após a remoção das stopwords sw_tese = [] sw_posInicial = [] aux_usu = [] sw_tagPosInicial = [] #texto marcado e sem stopwords sw_tagcomAce_posInicial = [] #texto COM ACENTOS marcado e sem stopwords #lista com dados após a aplicação de Stemming st_posInicial = [] st_tese = [] st_tagPosInicial = [] #texto marcado, sem stopwords e com stemmer aplicado st_tagcomAce_posInicial = [ ] #texto COM ACENTOS marcado, sem stopwords e com stemmer aplicado ############################################################################################################# #LISTA COM OS POSICIONAMENTOS INICIAIS APÓS APLICAÇÃO DA NORMALIZAÇAÕ posInicial_Normalizado = [] normalizacao = [] ############################################################################################################# #Aplicacao de Case Folding for d in dadosSql: dados.append([ re.sub('<[^>]*>', '', h.unescape(d[0])).lower(), re.sub('<[^>]*>', '', h.unescape(d[1])).lower() ]) for t in textotese: tese.append(re.sub('<[^>]*>', '', h.unescape(t[0])).lower()) #Colocando os textos de posicionamento inicial em numa lista separada for i in dados: x = 0 usu.append(i[x].upper()) posInicial.append( i[x + 1].lower() ) #lista com o posicionamento Inicial com todas as letras em minusculo ############################################################################################################# ### Classificacao das palavras de acordo com sua classe gramatical ### Utilizacao do postagger NLPNET ### http://nilc.icmc.usp.br/nlpnet/index.html# tagger = nlpnet.POSTagger() semAce_posInicial = [ ] #armazena o posInicial apenas sem acentos, sem pontuações, sem endereço web e sem numeros comAce_posInicial = [ ] #armazena o posInicial apenas COM acentos, sem pontuações, sem endereço web e sem numeros for i in posInicial: semAce_posInicial.append( removePontuacao(removeA(removeNum(removeSE(removeEndWeb((i))))))) for i in semAce_posInicial: tag_posInicial.append(tagger.tag(i)) for i in posInicial: comAce_posInicial.append( removePontuacao(removeNum(removeSE(removeEndWeb((i)))))) for i in comAce_posInicial: tag_comAce_posInicial.append(tagger.tag(i)) ############################################################################################################# #APENAS PARA REALIZAR TESTE E COLOCAR NA DISSERTACAO # pprint(semAce_posInicial) # pprint(comAce_posInicial) # exit() # tagg_posInicial = [] # for texto in posInicial: # tagg_posInicial.append(tagger.tag(texto)) # # print "posInicial" # pprint(posInicial) # # print "tagg_posInicial" # pprint(tagg_posInicial) ############################################################################################################# ############################################################################################################# ### REMOCAO DE STOPWORDS ### Remocao dos termos de acordo com a NLTK ### Remocao dos termos classificados como artigos, verbos, adverbios, etc... for i in usu: aux_usu.append(removeStopWords(i)) for i in tese: sw_tese.append(removeStopWords(i)) for i in posInicial: sw_posInicial.append(removeStopWords(i)) for i in tag_posInicial: sw_tagPosInicial.append(limpaCorpus(i)) for i in tag_comAce_posInicial: sw_tagcomAce_posInicial.append(limpaCorpus(i)) #################################################################################################################################### # Aplicação do RSPL Stemmer para remoção dos afixos das palavras da lingua portuguesa # Retirando afixos dos textos do posInicial e tese stemmer = RSLPStemmer() for i in range(len(sw_posInicial)): st_aux = sw_posInicial[i] string_aux = "" for sufixo in st_aux.split(): string_aux = string_aux + " " + stemmer.stem(sufixo) st_posInicial.append(string_aux) for i in range(len(sw_tese)): st_aux = sw_tese[i] string_aux = "" for sufixo in st_aux.split(): string_aux = string_aux + " " + stemmer.stem(sufixo) st_tese.append(string_aux) for i in range(len(sw_tagPosInicial)): termosST = "" auxST = [] for j in range(len(sw_tagPosInicial[i])): aux = stemmer.stem(sw_tagPosInicial[i][j][0]) etiqueta = sw_tagPosInicial[i][j][1] termosST = (aux, etiqueta) auxST.append(termosST) st_tagPosInicial.append(auxST) for i in range(len(sw_tagcomAce_posInicial)): termosST = "" auxST = [] for j in range(len(sw_tagcomAce_posInicial[i])): aux = stemmer.stem(sw_tagcomAce_posInicial[i][j][0]) etiqueta = sw_tagcomAce_posInicial[i][j][1] termosST = (aux, etiqueta) auxST.append(termosST) st_tagcomAce_posInicial.append(auxST) #################################################################################################################################### ### A NORMALIZACAO DE TERMOS REFERE-SE A TECNICA DE TROCAR PALAVRAS SINONIMAS, OU SEJA, QUE TENHAM SIGNIFICADO ## ### SEMELHANTE, POR UM UNICO TERMO REPRESENTATIVO NO CORPUS DE ANALISE. DESSA FORMA, É POSSIVEL AUMENTAR O GRAU ## ### DE SIMILARIDADE ENTRE OS TEXTOS ANALISADOS ATRAVES DO USO DE TECNICAS DE ANALISE ESTATISTICAS, COMO SIMILA ## ### RIDADE DE COSSENOS OU DISTANCIA EUCLIDIANA. ## #################################################################################################################################### ### A NORMALIZACAO FOI DESENVOLVIDA COM BASE NOS DADOS DISPONIBILIZADOS PELO PROJETO TEP 2.0 DO NILC/USP ## ### http://143.107.183.175:21480/tep2/index.htm ## ### ## ### FORMATO DO ARQUIVO ## ### NUM1. [Tipo] {termos sinonimos} <NUM2> ## ### 263. [Verbo] {consentir, deixar, permitir} <973> ## ### NUM1 = NUMERO DA LINHA DE REFERENCIA PARA TERMO SINONIMO ## ### NUM2 = NUMERO DA LINHA DE REFERENCIA PARA TERMO ANTONIMO (SENTIDO OPOSTO) ## #################################################################################################################################### #abre o arquivo com as relacoes de sinonimia (termos linhaWordNet) e antonimia (termos contrarios) #arquivo apenas com termos classificados como substantivos, adjetivos e verbos base_tep = codecs.open( os.path.join(os.path.dirname(__file__), '../base_tep2/base_tep.txt'), 'r', 'UTF8') # dicionario = open('/home/panceri/git/alpes_v1/base_tep2/dicionarioSinonimos.txt', 'w') #variavel com conteúdo do arquivo em memoria #não imprimir essa variável, MUITO GRANDEE!!! wordNet = base_tep.readlines() #fechar arquivo base_tep.close() #################################################################################################################################### ## NORMALIZAÇÃO FEITA COM BASE NOS RADICAIS DE FORMAÇÃO DAS PALAVRAS ## ## APLICAÇÃO DO RSPL PRIMEIRO PARA DEPOIS BUSCAR NA BASE OS TERMOS SIMILARES ## ## DENTRO DA BASE_TEP OS TERMOS TAMBÉM FORAM REDUZIDOS AOS SEUS RADICIAIS DE FORMAÇÃO ## ## O DICIONÁRIO ESTÁ COM A REFERÊNCIA PARA A LINHA AONDE ESTÃO OS TERMOS SINÔNIMOS ## ## OS TERMOS SÃO ANALISADOS CONSIDERANDO SUAS ACENTUAÇÕES, PARA APLICAÇÃO CORRETA DO RSLP ## #################################################################################################################################### yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() st_WordNetV = [ ] ##armazena num, tipo, e radical dos sinonimos - APENAS VERBOS st_WordNetN = [ ] ##armazena num, tipo, e radical dos sinonimos - APENAS SUBSTANTIVOS st_WordNetA = [ ] ##armazena num, tipo, e radical dos sinonimos - APENAS ADJETIVOS st_WordNetO = [ ] ##armazena num, tipo, e radical dos sinonimos - APENAS OUTROS for linhaWordnet in wordNet: listaAux = [] termos = re.findall(r"\{(.*)\}", linhaWordnet) num = re.findall(r"([0-9]+)\.", linhaWordnet) tipo = re.findall(r"\[(.*)\]", linhaWordnet) if tipo[0] == "Substantivo": listaAux.append(num) listaAux.append(tipo) for T in termos: aux = T.split() auxL = [] for i in aux: aux1 = i.replace(",", "") dadosStem = stemmer.stem(aux1) auxL.append(dadosStem) listaAux.append(auxL) st_WordNetN.append(listaAux) elif tipo[0] == "Verbo": listaAux.append(num) listaAux.append(tipo) for T in termos: aux = T.split() auxL = [] for i in aux: aux1 = i.replace(",", "") dadosStem = stemmer.stem(aux1) auxL.append(dadosStem) listaAux.append(auxL) st_WordNetV.append(listaAux) elif tipo[0] == "Adjetivo": listaAux.append(num) listaAux.append(tipo) for T in termos: aux = T.split() auxL = [] for i in aux: aux1 = i.replace(",", "") dadosStem = stemmer.stem(aux1) auxL.append(dadosStem) listaAux.append(auxL) st_WordNetA.append(listaAux) else: listaAux.append(num) listaAux.append(tipo) for T in termos: aux = T.split() auxL = [] for i in aux: aux1 = i.replace(",", "") dadosStem = stemmer.stem(aux1) auxL.append(dadosStem) listaAux.append(auxL) st_WordNetO.append(listaAux) duration = time.time() - start stats = yappi.get_func_stats() stats.save('stemmWordNet.out', type='callgrind') #################################################################################################################################### ### A ANÁLISE É REALIZADA COM BASE NO TEXTO SEM A EXCLUSÃO DOS ACENTOS ## ### POIS AO EXCLUÍ-LOS A REDUÇÃO AO RADICAL DE FORMAÇÃO (APLICAÇÃO DO RSLP) É PREJUDICADA ## ### OS TESTES REALIZADOS MOSTRARAM QUE ESSA É UMA MELHOR ABORDAGEM, UMA VEZ QUE NOSSOS TEXTOS SÃO PEQUENOS ## ### E PRECISAMOS CHEGAR O MAIS PRÓXIMO POSSÍVEL SEM CONSIDERAR SEUS SENTIDOS E/OU CONTEXTOS ## #################################################################################################################################### yappi.set_clock_type('cpu') yappi.start(builtins=True) start = time.time() normalizacao = normalizacaoWordnet(st_WordNetA, st_WordNetN, st_WordNetV, st_WordNetO, st_tagcomAce_posInicial) ############################################################### # Colocando os textos normalizados numa lista de 1 diemensão ############################################################### stringNorm = "" auxNorm = [] for i in range(len(normalizacao)): auxNorm = normalizacao[i] for x in range(len(auxNorm)): stringNorm = stringNorm + " " + auxNorm[x] posInicial_Normalizado.append(stringNorm) stringNorm = "" duration = time.time() - start stats = yappi.get_func_stats() stats.save('normalizacaoWordnet.out', type='callgrind') #################################################################################################################################### # print "posInicial" # pprint(posInicial) # # print "comAce_posInicial" # pprint(comAce_posInicial) # # print "tag_comAce_posInicial" # pprint(tag_comAce_posInicial) # # print "sw_tagcomAce_posInicial" # pprint(sw_tagcomAce_posInicial) # # print "st_tagcomAce_posInicial" # pprint(st_tagcomAce_posInicial) # print "posInicial_Normalizado" # print len(posInicial_Normalizado) # pprint(posInicial_Normalizado) # exit() #################################################################################################################################### #retorno da função - usado na views.py para alimentar o template debate.html #passar parametros que devem ser apresentados na templates debate.html return [ st_tese, posInicial, sw_tese, aux_usu, st_posInicial, tese, posInicial_Normalizado ]
def Apriori(self): tStart = time.time() candidate_num, frequnt_num, all_frequent_itemsets = self.Find_frequent_one( self.sup) print('L1====================================') print('Frequent 1-itemset is', frequnt_num) process = psutil.Process(os.getpid()) print('Used Memory:', process.memory_info().rss / 1024 / 1024, 'MB') prev_frequent_num = frequnt_num print('====================================') tEnd = time.time() print(tEnd - tStart) prev_frequent = [x for x in all_frequent_itemsets] prev_frequent.sort(key=lambda tup: tup) length = 2 while len(prev_frequent) > 1: tStart = time.time() if self.debug: # profilling yappi.clear_stats() # clear profiler yappi.set_clock_type('cpu') yappi.start(builtins=True) # track builtins new_candidate = self.Generate_candidate(prev_frequent, length) print('L%s====================================' % (length)) print('Total candidates is ', len(new_candidate) + prev_frequent_num) self.htree = self.Generate_hash_tree(new_candidate) self.Frequent_support(length) # find frequent itemsets new_frequent = self.htree.get_frequent_itemsets(self.sup) print('Frequent itemset is', len(new_frequent) + prev_frequent_num) process = psutil.Process(os.getpid()) print('Used Memory:', process.memory_info().rss / 1024 / 1024, 'MB') print('====================================') tEnd = time.time() print(tEnd - tStart) prev_frequent_num = len(new_frequent) + prev_frequent_num # all_frequent_itemsets.extend(new_frequent) prev_frequent = [tup[0] for tup in new_frequent] prev_frequent.sort() if self.debug: yappi.stop() stat = yappi.get_func_stats() var = 'callgrind.generate_tree' + str(length) stat.save(var, type='callgrind') # profilling end length += 1 ''' gc ''' del self.htree del new_candidate del new_frequent gc.collect() return all_frequent_itemsets
def __init__(self, args=None): QtCore.QObject.__init__(self) self.app = QtGui.QApplication(["FireSim"]) self.args = args self.config = Config("data/config.json") if self.args.profile: try: import yappi yappi.start() except ImportError: log.error("Could not enable YaPPI profiling") self._selected_fixture_strand = 0 self._selected_fixture_address = 0 self._selected_fixture_pixels = 0 self.selected_fixture = None self.is_blurred = False self.scene = Scene( os.path.join(self.config.get("scene_root"), self.args.scene) + ".json") self.scenecontroller = SceneController(app=self, scene=self.scene) QtDeclarative.qmlRegisterType(CanvasWidget, "FireSim", 1, 0, "SimCanvas") QtDeclarative.qmlRegisterType(FixtureWidget, "FireSim", 1, 0, "Fixture") self.view = QtDeclarative.QDeclarativeView() self.view.setWindowTitle("FireSim") self.view.setResizeMode( QtDeclarative.QDeclarativeView.SizeRootObjectToView) self.view.closeEvent = self.on_close self.context = self.view.rootContext() self.context.setContextProperty('main', self) self.fixture_info_list = [] self.context.setContextProperty('fixtureInfoModel', self.fixture_info_list) self.view.setSource(QtCore.QUrl('ui/qml/FireSimGUI.qml')) self.root = self.view.rootObject() self.item_frame = self.root.findChild(QtDeclarative.QDeclarativeItem) self.canvas = self.root.findChild(CanvasWidget) self.canvas.gui = self cw, ch = self.scenecontroller.scene.extents() self.canvas.setWidth(cw) self.canvas.setHeight(ch) self.scenecontroller.set_canvas(self.canvas) self.root.backdrop_showhide_callback.connect( self.on_btn_backdrop_showhide) self.root.labels_showhide_callback.connect(self.on_btn_labels_showhide) self.root.lock_callback.connect(self.on_btn_lock) self.root.show_center_callback.connect(self.on_btn_show_center) self.root.toggle_blurred_callback.connect(self.on_btn_toggle_blurred) #self.net_thread = QtCore.QThread() #self.net_thread.start() self.netcontroller = NetController(self) #self.netcontroller.moveToThread(self.net_thread) #self.netcontroller.start.emit() self.net_stats_timer = QtCore.QTimer() self.net_stats_timer.setInterval(1000) self.net_stats_timer.timeout.connect(self.update_net_stats) self.net_stats_timer.start() self.netcontroller.data_received.connect(self.on_network_event) self.scenecontroller.new_frame.connect( self.netcontroller.frame_complete) self.netcontroller.data_received.connect( self.scenecontroller.process_command) self.view.setFixedSize(max(640, cw + 130), max(480, ch)) log.info("FireSimGUI Ready.") self.view.show()
def test_merge_stats(self): _timings = { "a_1": 15, "b_1": 14, "c_1": 12, "d_1": 10, "e_1": 9, "f_1": 7, "g_1": 6, "h_1": 5, "i_1": 1 } _yappi._set_test_timings(_timings) def a(): b() def b(): c() def c(): d() def d(): e() def e(): f() def f(): g() def g(): h() def h(): i() def i(): pass yappi.start() a() a() yappi.stop() stats = yappi.get_func_stats() self.assertRaises(NotImplementedError, stats.save, "", "INVALID_SAVE_TYPE") stats.save("ystats2.ys") yappi.clear_stats() _yappi._set_test_timings(_timings) yappi.start() a() stats = yappi.get_func_stats().add("ystats2.ys") fsa = utils.find_stat_by_name(stats, "a") fsb = utils.find_stat_by_name(stats, "b") fsc = utils.find_stat_by_name(stats, "c") fsd = utils.find_stat_by_name(stats, "d") fse = utils.find_stat_by_name(stats, "e") fsf = utils.find_stat_by_name(stats, "f") fsg = utils.find_stat_by_name(stats, "g") fsh = utils.find_stat_by_name(stats, "h") fsi = utils.find_stat_by_name(stats, "i") self.assertEqual(fsa.ttot, 45) self.assertEqual(fsa.ncall, 3) self.assertEqual(fsa.nactualcall, 3) self.assertEqual(fsa.tsub, 3) self.assertEqual(fsa.children[fsb].ttot, fsb.ttot) self.assertEqual(fsa.children[fsb].tsub, fsb.tsub) self.assertEqual(fsb.children[fsc].ttot, fsc.ttot) self.assertEqual(fsb.children[fsc].tsub, fsc.tsub) self.assertEqual(fsc.tsub, 6) self.assertEqual(fsc.children[fsd].ttot, fsd.ttot) self.assertEqual(fsc.children[fsd].tsub, fsd.tsub) self.assertEqual(fsd.children[fse].ttot, fse.ttot) self.assertEqual(fsd.children[fse].tsub, fse.tsub) self.assertEqual(fse.children[fsf].ttot, fsf.ttot) self.assertEqual(fse.children[fsf].tsub, fsf.tsub) self.assertEqual(fsf.children[fsg].ttot, fsg.ttot) self.assertEqual(fsf.children[fsg].tsub, fsg.tsub) self.assertEqual(fsg.ttot, 18) self.assertEqual(fsg.tsub, 3) self.assertEqual(fsg.children[fsh].ttot, fsh.ttot) self.assertEqual(fsg.children[fsh].tsub, fsh.tsub) self.assertEqual(fsh.ttot, 15) self.assertEqual(fsh.tsub, 12) self.assertEqual(fsh.tavg, 5) self.assertEqual(fsh.children[fsi].ttot, fsi.ttot) self.assertEqual(fsh.children[fsi].tsub, fsi.tsub)
def start(self): yappi.start() return '<a href="stats">stats</a>'
def test_merge_aabab_aabbc(self): _timings = { "a_1": 15, "a_2": 14, "b_1": 12, "a_3": 10, "b_2": 9, "c_1": 4 } _yappi._set_test_timings(_timings) def a(): if self._ncall == 1: self._ncall += 1 a() elif self._ncall == 5: self._ncall += 1 a() else: b() def b(): if self._ncall == 2: self._ncall += 1 a() elif self._ncall == 6: self._ncall += 1 b() elif self._ncall == 7: c() else: return def c(): pass self._ncall = 1 stats = utils.run_and_get_func_stats(a, ) stats.save("ystats1.ys") yappi.clear_stats() _yappi._set_test_timings(_timings) #stats.print_all() self._ncall = 5 stats = utils.run_and_get_func_stats(a, ) stats.save("ystats2.ys") #stats.print_all() def a(): # same name but another function(code object) pass yappi.start() a() stats = yappi.get_func_stats().add(["ystats1.ys", "ystats2.ys"]) #stats.print_all() self.assertEqual(len(stats), 4) fsa = None for stat in stats: if stat.name == "a" and stat.ttot == 45: fsa = stat break self.assertTrue(fsa is not None) self.assertEqual(fsa.ncall, 7) self.assertEqual(fsa.nactualcall, 3) self.assertEqual(fsa.ttot, 45) self.assertEqual(fsa.tsub, 10) fsb = utils.find_stat_by_name(stats, "b") fsc = utils.find_stat_by_name(stats, "c") self.assertEqual(fsb.ncall, 6) self.assertEqual(fsb.nactualcall, 3) self.assertEqual(fsb.ttot, 36) self.assertEqual(fsb.tsub, 27) self.assertEqual(fsb.tavg, 6) self.assertEqual(fsc.ttot, 8) self.assertEqual(fsc.tsub, 8) self.assertEqual(fsc.tavg, 4) self.assertEqual(fsc.nactualcall, fsc.ncall, 2)
def test_subsequent_profile(self): WORKER_COUNT = 5 def a(): pass def b(): pass def c(): pass _timings = { "a_1": 3, "b_1": 2, "c_1": 1, } yappi.start() def g(): pass g() yappi.stop() yappi.clear_stats() _yappi._set_test_timings(_timings) yappi.start() _dummy = [] for i in range(WORKER_COUNT): t = threading.Thread(target=a) t.start() t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=b) t.start() _dummy.append(t) t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=a) t.start() t.join() for i in range(WORKER_COUNT): t = threading.Thread(target=c) t.start() t.join() yappi.stop() yappi.start() def f(): pass f() stats = yappi.get_func_stats() fsa = utils.find_stat_by_name(stats, 'a') fsb = utils.find_stat_by_name(stats, 'b') fsc = utils.find_stat_by_name(stats, 'c') self.assertEqual(fsa.ncall, 10) self.assertEqual(fsb.ncall, 5) self.assertEqual(fsc.ncall, 5) self.assertEqual(fsa.ttot, fsa.tsub, 30) self.assertEqual(fsb.ttot, fsb.tsub, 10) self.assertEqual(fsc.ttot, fsc.tsub, 5) # MACOSx optimizes by only creating one worker thread self.assertTrue(len(yappi.get_thread_stats()) >= 2)
def test_ctx_stats(self): from threading import Thread DUMMY_WORKER_COUNT = 5 yappi.start() class DummyThread(Thread): pass def dummy(): pass def dummy_worker(): pass for i in range(DUMMY_WORKER_COUNT): t = DummyThread(target=dummy_worker) t.start() t.join() yappi.stop() stats = yappi.get_thread_stats() tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is not None) yappi.clear_stats() time.sleep(1.0) _timings = { "a_1": 6, "b_1": 5, "c_1": 3, "d_1": 1, "a_2": 4, "b_2": 3, "c_2": 2, "d_2": 1 } _yappi._set_test_timings(_timings) class Thread1(Thread): pass class Thread2(Thread): pass def a(): b() def b(): c() def c(): d() def d(): time.sleep(0.6) yappi.set_clock_type("wall") yappi.start() t1 = Thread1(target=a) t1.start() t2 = Thread2(target=a) t2.start() t1.join() t2.join() stats = yappi.get_thread_stats() # the fist clear_stats clears the context table? tsa = utils.find_stat_by_name(stats, "DummyThread") self.assertTrue(tsa is None) tst1 = utils.find_stat_by_name(stats, "Thread1") tst2 = utils.find_stat_by_name(stats, "Thread2") tsmain = utils.find_stat_by_name(stats, "_MainThread") dummy() # call dummy to force ctx name to be retrieved again. self.assertTrue(len(stats) == 3) self.assertTrue(tst1 is not None) self.assertTrue(tst2 is not None) # TODO: I put dummy() to fix below, remove the comments after a while. self.assertTrue( # FIX: I see this fails sometimes tsmain is not None, 'Could not find "_MainThread". Found: %s' % (', '.join(utils.get_stat_names(stats)))) self.assertTrue(1.0 > tst2.ttot >= 0.5) self.assertTrue(1.0 > tst1.ttot >= 0.5) # test sorting of the ctx stats stats = stats.sort("totaltime", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot >= stat.ttot) prev_stat = stat stats = stats.sort("totaltime", "asc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.ttot <= stat.ttot) prev_stat = stat stats = stats.sort("schedcount", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.sched_count >= stat.sched_count) prev_stat = stat stats = stats.sort("name", "desc") prev_stat = None for stat in stats: if prev_stat: self.assertTrue(prev_stat.name >= stat.name) prev_stat = stat self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sorttype_arg") self.assertRaises(yappi.YappiError, stats.sort, "invalid_thread_sortorder_arg")
def _on_keyboard_down(self, _keyboard, keycode, _text, modifiers): self.last_key_down = keycode ctrl_pressed = "ctrl" in modifiers if self.controls.note.focus: return # when making notes, don't allow keyboard shortcuts popup = self.popup_open if popup: if keycode[1] in ["f5", "f6", "f7", "f8"]: # switch between popups popup.dismiss() return elif keycode[1] in ["enter", "numpadenter"]: fn = getattr(popup.content, "on_submit", None) if fn: fn() return else: return shift_pressed = "shift" in modifiers shortcuts = self.shortcuts if keycode[1] == "spacebar": self.toggle_continuous_analysis() elif keycode[1] == "k": self.board_gui.toggle_coordinates() elif keycode[1] in ["pause", "break", "f15"] and not ctrl_pressed: self.controls.timer.paused = not self.controls.timer.paused elif keycode[1] in ["`", "~", "f12"]: self.zen = (self.zen + 1) % 3 elif keycode[1] in ["left", "z"]: self("undo", 1 + shift_pressed * 9 + ctrl_pressed * 9999) elif keycode[1] in ["right", "x"]: self("redo", 1 + shift_pressed * 9 + ctrl_pressed * 9999) elif keycode[1] == "home": self("undo", 9999) elif keycode[1] == "end": self("redo", 9999) elif keycode[1] == "pageup": self.controls.move_tree.make_selected_node_main_branch() elif keycode[1] == "n" and not ctrl_pressed: self("find-mistake", "undo" if shift_pressed else "redo") elif keycode[1] == "delete" and ctrl_pressed: self.controls.move_tree.delete_selected_node() elif keycode[1] == "c" and not ctrl_pressed: self.controls.move_tree.toggle_selected_node_collapse() elif keycode[1] == "n" and ctrl_pressed: self("new-game-popup") elif keycode[1] == "l" and ctrl_pressed: self("analyze-sgf-popup") elif keycode[1] == "s" and ctrl_pressed: self("save-game") elif keycode[1] == "d" and ctrl_pressed: self("save-game-as-popup") elif keycode[1] == "c" and ctrl_pressed: Clipboard.copy(self.game.root.sgf()) self.controls.set_status(i18n._("Copied SGF to clipboard."), STATUS_INFO) elif keycode[1] == "v" and ctrl_pressed: self.load_sgf_from_clipboard() elif keycode[1] == "b" and shift_pressed: self("undo", "main-branch") elif keycode[1] in shortcuts.keys() and not ctrl_pressed: shortcut = shortcuts[keycode[1]] if isinstance(shortcut, Widget): shortcut.trigger_action(duration=0) else: self(*shortcut) elif keycode[1] == "f9" and self.debug_level >= OUTPUT_EXTRA_DEBUG: import yappi yappi.set_clock_type("cpu") yappi.start() self.log("starting profiler", OUTPUT_ERROR) elif keycode[1] == "f10" and self.debug_level >= OUTPUT_EXTRA_DEBUG: import time import yappi stats = yappi.get_func_stats() filename = f"callgrind.{int(time.time())}.prof" stats.save(filename, type="callgrind") self.log(f"wrote profiling results to {filename}", OUTPUT_ERROR)
def profile(config_location="/etc/redeem"): import yappi yappi.start() main(config_location) yappi.get_func_stats().print_all()
def filter_opencv(im): for i in range(100000): im += 1 return im def filter_homemade(im): for i in range(100000): im += 2 return im import yappi filename = 'callgrind.filename.prof' yappi.set_clock_type('cpu') yappi.start(builtins=True) filter_homemade(0) filter_opencv(0) stats = yappi.get_func_stats() stats.save(filename, type='callgrind')
def profile(): import yappi yappi.start() main() yappi.get_func_stats().print_all()
def test_basic(self): @asyncio.coroutine def a(): yield from async_sleep(0.1) burn_cpu(0.2) @asyncio.coroutine def b(): yield from a() @asyncio.coroutine def recursive_a(n): if not n: return burn_io(0.1) yield from async_sleep(0.1) yield from recursive_a(n - 1) def tag_cbk(): cthread = threading.current_thread() try: return cthread._tag except: return -1 yappi.set_clock_type("wall") threading.current_thread()._tag = 0 yappi.set_tag_callback(tag_cbk) def _thread_event_loop(loop): asyncio.set_event_loop(loop) loop.run_forever() _TCOUNT = 3 _ctag = 1 ts = [] for i in range(_TCOUNT): _loop = asyncio.new_event_loop() t = threading.Thread(target=_thread_event_loop, args=(_loop, )) t._tag = _ctag t._loop = _loop t.start() ts.append(t) _ctag += 1 @asyncio.coroutine def stop_loop(): asyncio.get_event_loop().stop() @asyncio.coroutine def driver(): futs = [] fut = asyncio.run_coroutine_threadsafe(a(), ts[0]._loop) futs.append(fut) fut = asyncio.run_coroutine_threadsafe(recursive_a(5), ts[1]._loop) futs.append(fut) fut = asyncio.run_coroutine_threadsafe(b(), ts[2]._loop) futs.append(fut) for fut in futs: fut.result() # stop asyncio loops in threads for t in ts: asyncio.run_coroutine_threadsafe(stop_loop(), t._loop) yappi.start() asyncio.get_event_loop().run_until_complete(driver()) yappi.stop() traces = yappi.get_func_stats() t1 = ''' tests/test_asyncio.py:140 driver 1 0.000022 1.015813 1.015813 ..ts/test_asyncio.py:103 recursive_a 6/1 0.000071 1.014597 0.169099 async_sleep 7 0.000000 0.714345 0.102049 tests/test_asyncio.py:94 a 2 0.000025 0.610155 0.305078 tests/utils.py:135 burn_io 5 0.000018 0.505879 0.101176 tests/utils.py:126 burn_cpu 2 0.380813 0.404432 0.202216 tests/test_asyncio.py:99 b 1 0.000005 0.305355 0.305355 ''' self.assert_traces_almost_equal(t1, traces) traces = yappi.get_func_stats(filter={'tag': 2}) t1 = ''' ..ts/test_asyncio.py:103 recursive_a 6/1 0.000160 1.062948 0.177158 async_sleep 5 0.000000 0.561803 0.112361 tests/utils.py:135 burn_io 5 0.000035 0.500985 0.100197 ''' self.assert_traces_almost_equal(t1, traces)