def profiled_func(*args, **kwargs): try: profile = pyinstrument.Profiler() except: profile = pyinstrument.Profiler(use_signal=False) try: profile.start() result = func(*args, **kwargs) profile.stop() return result finally: print(profile.output_text(color=True))
def from_pyinstrument_trace(file_ref): import pyinstrument profiler = pyinstrument.Profiler() profiler.add(file_ref) root_frame = profiler.root_frame() def visit_frame(frame): code_pos = frame.code_position_short if code_pos is None: code_pos = '' index = code_pos.find('site-packages') if index > 0: code_pos = code_pos[index + (len('site-packages') + 1):] measurement = Measurement( '{function} {code_pos}'.format(function=frame.function or '', code_pos=code_pos), self_time=datetime.timedelta(seconds=frame.self_time)) for child_frame in frame.children: measurement.add_child(visit_frame(child_frame)) return measurement return visit_frame(root_frame)
def time_pyinstrument(function, repeats): timer = Timer(stmt=function) p = pyinstrument.Profiler() p.start() result = timer.repeat(number=repeats) p.stop() return result
def main(): opt = get_opt() if opt.dbg: util.do_dbg = True # just list? if opt.list: descriptors.list_descriptors() return # just check stat_data.py? if opt.check_wt_stat_data: descriptors.check_wt_stat_data() return if opt.profile: # pip install -e git+https://github.com/joerick/pyinstrument.git#egg=pyinstrument # pylint: disable=import-error import pyinstrument, codecs p = pyinstrument.Profiler() p.start() flow.main(opt) p.stop() out = codecs.getwriter('UTF-8')(sys.stderr) out.write(p.output_text(unicode=True, color=True)) else: flow.main(opt)
def profile_if_necessary(profiler_name, file_path): if profiler_name == 'pyinstrument': profiler = pyinstrument.Profiler(use_signal=False) profiler.start() try: yield finally: if profiler_name == 'pyinstrument': profiler.stop() profiler.save(filename=os.path.join(file_path, 'launch-task.trace'))
def main(): p = pyinstrument.Profiler() p.start() t_lz4 = timeit.timeit(lambda: StoryData.decode(StoryData( random_content(), version=StoryData.VERSION_LZ4).encode()), number=10000) print(t_lz4) p.stop() html = p.output_html() with open('benchmark_story_data_lz4.html', 'w') as f: f.write(html) p = pyinstrument.Profiler() p.start() t_gzip = timeit.timeit(lambda: StoryData.decode(StoryData( random_content(), version=StoryData.VERSION_GZIP).encode()), number=10000) print(t_gzip) p.stop() html = p.output_html() with open('benchmark_story_data_gzip.html', 'w') as f: f.write(html)
def start(self) -> None: """ Begin profiling. """ self.start_time = time.time() self._profiler = cProfile.Profile() self._callstack_profiler = pyinstrument.Profiler() self._profiler.enable() self._callstack_profiler.start()
def profile_solution(n_clicks: int, name: str) -> int: if not name: return {} module = sys.modules[__name__] func = getattr(module, name) print(inspect.getsource(func)) with pi.Profiler() as profiler: func() profile_str = profiler.output(renderer=pi.renderers.JSONRenderer( show_all=False)) profile_dict = json.loads(s=profile_str) return profile_dict
def start_profiler(): import pyinstrument profiler = pyinstrument.Profiler() profiler.start() return profiler
timings = g.collections.defaultdict(dict) iterations = 10 repeats = 3 for file_name in file_names: for test_name, test_string in tests.items(): time_min = min(timeit.repeat(test_string, setup.format(file_name), repeat=repeats, number=iterations)) timings[test_name][file_name] = time_min / iterations result = {} result['cpu_info'] = subprocess.check_output(['cat', '/proc/cpuinfo']) result['baseline'] = baseline.tolist() result['timestamp'] = time.time() result['timings'] = timings ''' import pyinstrument profiler = pyinstrument.Profiler() profiler.start() typical_application() profiler.stop() print(profiler.output_text(unicode=True, color=True))
def run(self, **kwargs): """Main routine of the model. Arguments: kwargs (:obj:`dict`): """ for arg, val in kwargs.items(): setattr(self, arg, val) with self.timers["setup"]: self.setup() logging.info("Reading restarts:") restart.read_restart(self.itt) diagnostics.read_restart(self) self.enditt = self.itt + int(self.runlen / self.dt_tracer) logging.info("Starting integration for {:.2e}s".format(self.runlen)) logging.info(" from time step {} to {}".format(self.itt,self.enditt)) try: while self.itt < self.enditt: if self.itt == 3 and self.profile_mode: # when using bohrium, most kernels should be pre-compiled # after three iterations import pyinstrument profiler = pyinstrument.Profiler() profiler.start() with self.timers["main"]: self.set_forcing() if self.enable_idemix: idemix.set_idemix_parameter(self) if self.enable_idemix_M2 or self.enable_idemix_niw: idemix.set_spectral_parameter(self) eke.set_eke_diffusivities(self) tke.set_tke_diffusivities(self) with self.timers["momentum"]: momentum.momentum(self) with self.timers["temperature"]: thermodynamics.thermodynamics(self) if self.enable_eke or self.enable_tke or self.enable_idemix: advection.calculate_velocity_on_wgrid(self) with self.timers["eke"]: if self.enable_eke: eke.integrate_eke(self) with self.timers["idemix"]: if self.enable_idemix_M2: idemix.integrate_idemix_M2(self) if self.enable_idemix_niw: idemix.integrate_idemix_niw(self) if self.enable_idemix: idemix.integrate_idemix(self) if self.enable_idemix_M2 or self.enable_idemix_niw: idemix.wave_interaction(self) with self.timers["tke"]: if self.enable_tke: tke.integrate_tke(self) if self.enable_cyclic_x: cyclic.setcyclic_x(self.u[:,:,:,self.taup1]) cyclic.setcyclic_x(self.v[:,:,:,self.taup1]) if self.enable_tke: cyclic.setcyclic_x(self.tke[:,:,:,self.taup1]) if self.enable_eke: cyclic.setcyclic_x(self.eke[:,:,:,self.taup1]) if self.enable_idemix: cyclic.setcyclic_x(self.E_iw[:,:,:,self.taup1]) if self.enable_idemix_M2: cyclic.setcyclic_x(self.E_M2[:,:,:,self.taup1]) if self.enable_idemix_niw: cyclic.setcyclic_x(self.E_niw[:,:,:,self.taup1]) # diagnose vertical velocity at taup1 if self.enable_hydrostatic: momentum.vertical_velocity(self) self.flush() with self.timers["diagnostics"]: diagnostics.sanity_check(self) if self.enable_neutral_diffusion and self.enable_skew_diffusion: isoneutral.isoneutral_diag_streamfunction(self) diagnostics.diagnose(self) # shift time otaum1 = self.taum1 self.taum1 = self.tau self.tau = self.taup1 self.taup1 = otaum1 self.itt += 1 logging.info("Current iteration: {}".format(self.itt)) logging.debug("Time step took {}s".format(self.timers["main"].getLastTime())) except: diagnostics.panic_output(self) raise finally: logging.debug("Timing summary:") logging.debug(" setup time summary = {}s".format(self.timers["setup"].getTime())) logging.debug(" main loop time summary = {}s".format(self.timers["main"].getTime())) logging.debug(" momentum = {}s".format(self.timers["momentum"].getTime())) logging.debug(" pressure = {}s".format(self.timers["pressure"].getTime())) logging.debug(" friction = {}s".format(self.timers["friction"].getTime())) logging.debug(" thermodynamics = {}s".format(self.timers["temperature"].getTime())) logging.debug(" lateral mixing = {}s".format(self.timers["isoneutral"].getTime())) logging.debug(" vertical mixing = {}s".format(self.timers["vmix"].getTime())) logging.debug(" equation of state = {}s".format(self.timers["eq_of_state"].getTime())) logging.debug(" EKE = {}s".format(self.timers["eke"].getTime())) logging.debug(" IDEMIX = {}s".format(self.timers["idemix"].getTime())) logging.debug(" TKE = {}s".format(self.timers["tke"].getTime())) logging.debug(" diagnostics and I/O = {}s".format(self.timers["diagnostics"].getTime())) if self.profile_mode: try: profiler.stop() with open("profile.html", "w") as f: f.write(profiler.output_html()) except UnboundLocalError: # profiler has not been started pass
def test_corrected_jets_factory(): import os from coffea.jetmet_tools import CorrectedJetsFactory, CorrectedMETFactory, JECStack events = None from coffea.nanoevents import NanoEventsFactory factory = NanoEventsFactory.from_root( os.path.abspath("tests/samples/nano_dy.root")) events = factory.events() jec_stack_names = [ "Summer16_23Sep2016V3_MC_L1FastJet_AK4PFPuppi", "Summer16_23Sep2016V3_MC_L2Relative_AK4PFPuppi", "Summer16_23Sep2016V3_MC_L2L3Residual_AK4PFPuppi", "Summer16_23Sep2016V3_MC_L3Absolute_AK4PFPuppi", "Spring16_25nsV10_MC_PtResolution_AK4PFPuppi", "Spring16_25nsV10_MC_SF_AK4PFPuppi", ] for key in evaluator.keys(): if "Summer16_23Sep2016V3_MC_UncertaintySources_AK4PFPuppi" in key: jec_stack_names.append(key) jec_inputs = {name: evaluator[name] for name in jec_stack_names} jec_stack = JECStack(jec_inputs) name_map = jec_stack.blank_name_map name_map["JetPt"] = "pt" name_map["JetMass"] = "mass" name_map["JetEta"] = "eta" name_map["JetA"] = "area" jets = events.Jet jets["pt_raw"] = (1 - jets["rawFactor"]) * jets["pt"] jets["mass_raw"] = (1 - jets["rawFactor"]) * jets["mass"] jets["pt_gen"] = ak.values_astype(ak.fill_none(jets.matched_gen.pt, 0), np.float32) jets["rho"] = ak.broadcast_arrays(events.fixedGridRhoFastjetAll, jets.pt)[0] name_map["ptGenJet"] = "pt_gen" name_map["ptRaw"] = "pt_raw" name_map["massRaw"] = "mass_raw" name_map["Rho"] = "rho" jec_cache = cachetools.Cache(np.inf) print(name_map) tic = time.time() jet_factory = CorrectedJetsFactory(name_map, jec_stack) toc = time.time() print("setup corrected jets time =", toc - tic) tic = time.time() prof = pyinstrument.Profiler() prof.start() corrected_jets = jet_factory.build(jets, lazy_cache=jec_cache) prof.stop() toc = time.time() print("corrected_jets build time =", toc - tic) print(prof.output_text(unicode=True, color=True, show_all=True)) tic = time.time() print("Generated jet pt:", corrected_jets.pt_gen) print("Original jet pt:", corrected_jets.pt_orig) print("Raw jet pt:", jets.pt_raw) print("Corrected jet pt:", corrected_jets.pt) print("Original jet mass:", corrected_jets.mass_orig) print("Raw jet mass:", jets["mass_raw"]) print("Corrected jet mass:", corrected_jets.mass) print("jet eta:", jets.eta) for unc in jet_factory.uncertainties(): print(unc) print(corrected_jets[unc].up.pt) print(corrected_jets[unc].down.pt) toc = time.time() print("build all jet variations =", toc - tic) # Test that the corrections were applied correctly from coffea.jetmet_tools import ( FactorizedJetCorrector, JetResolution, JetResolutionScaleFactor, ) scalar_form = ak.without_parameters(jets["pt_raw"]).layout.form corrector = FactorizedJetCorrector( **{name: evaluator[name] for name in jec_stack_names[0:4]}) corrs = corrector.getCorrection(JetEta=jets["eta"], Rho=jets["rho"], JetPt=jets["pt_raw"], JetA=jets["area"]) reso = JetResolution( **{name: evaluator[name] for name in jec_stack_names[4:5]}) jets["jet_energy_resolution"] = reso.getResolution( JetEta=jets["eta"], Rho=jets["rho"], JetPt=jets["pt_raw"], form=scalar_form, lazy_cache=jec_cache, ) resosf = JetResolutionScaleFactor( **{name: evaluator[name] for name in jec_stack_names[5:6]}) jets["jet_energy_resolution_scale_factor"] = resosf.getScaleFactor( JetEta=jets["eta"], lazy_cache=jec_cache) # Filter out the non-deterministic (no gen pt) jets def smear_factor(jetPt, pt_gen, jersf): return (ak.full_like(jetPt, 1.0) + (jersf[:, 0] - ak.full_like(jetPt, 1.0)) * (jetPt - pt_gen) / jetPt) test_gen_pt = ak.concatenate( [corrected_jets.pt_gen[0, :-2], corrected_jets.pt_gen[-1, :-1]]) test_raw_pt = ak.concatenate([jets.pt_raw[0, :-2], jets.pt_raw[-1, :-1]]) test_pt = ak.concatenate( [corrected_jets.pt[0, :-2], corrected_jets.pt[-1, :-1]]) test_eta = ak.concatenate([jets.eta[0, :-2], jets.eta[-1, :-1]]) test_jer = ak.concatenate([ jets.jet_energy_resolution[0, :-2], jets.jet_energy_resolution[-1, :-1] ]) test_jer_sf = ak.concatenate([ jets.jet_energy_resolution_scale_factor[0, :-2], jets.jet_energy_resolution_scale_factor[-1, :-1], ]) test_jec = ak.concatenate([corrs[0, :-2], corrs[-1, :-1]]) test_corrected_pt = ak.concatenate( [corrected_jets.pt[0, :-2], corrected_jets.pt[-1, :-1]]) test_corr_pt = test_raw_pt * test_jec test_pt_smear_corr = test_corr_pt * smear_factor(test_corr_pt, test_gen_pt, test_jer_sf) # Print the results of the "by-hand" calculations and confirm that the values match the expected values print("\nConfirm the CorrectedJetsFactory values:") print("Jet pt (gen)", test_gen_pt.tolist()) print("Jet pt (raw)", test_raw_pt.tolist()) print("Jet pt (nano):", test_pt.tolist()) print("Jet eta:", test_eta.tolist()) print("Jet energy resolution:", test_jer.tolist()) print("Jet energy resolution sf:", test_jer_sf.tolist()) print("Jet energy correction:", test_jec.tolist()) print("Corrected jet pt (ref)", test_corr_pt.tolist()) print("Corrected & smeared jet pt (ref):", test_pt_smear_corr.tolist()) print("Corrected & smeared jet pt:", test_corrected_pt.tolist(), "\n") assert ak.all(np.abs(test_pt_smear_corr - test_corrected_pt) < 1e-6) name_map["METpt"] = "pt" name_map["METphi"] = "phi" name_map["JetPhi"] = "phi" name_map["UnClusteredEnergyDeltaX"] = "MetUnclustEnUpDeltaX" name_map["UnClusteredEnergyDeltaY"] = "MetUnclustEnUpDeltaY" tic = time.time() met_factory = CorrectedMETFactory(name_map) toc = time.time() print("setup corrected MET time =", toc - tic) met = events.MET tic = time.time() # prof = pyinstrument.Profiler() # prof.start() corrected_met = met_factory.build(met, corrected_jets, lazy_cache=jec_cache) # prof.stop() toc = time.time() # print(prof.output_text(unicode=True, color=True, show_all=True)) print("corrected_met build time =", toc - tic) tic = time.time() print(corrected_met.pt_orig) print(corrected_met.pt) prof = pyinstrument.Profiler() prof.start() for unc in jet_factory.uncertainties() + met_factory.uncertainties(): print(unc) print(corrected_met[unc].up.pt) print(corrected_met[unc].down.pt) prof.stop() toc = time.time() print("build all met variations =", toc - tic) print(prof.output_text(unicode=True, color=True, show_all=True))
def main(): term = Terminal() if sys.version_info < (2, 7): raise RuntimeError( "Jirafs requires minimally version 2.7 of Python 2, or " "any version of Python 3. Please upgrade your version of " "python before using Jirafs.") if utils.get_git_version() < LooseVersion("1.8"): raise RuntimeError( "Jirafs requires minimally version 1.8 of Git. Please " "upgrade your version of git before using Jirafs.") commands = utils.get_installed_commands() parser = argparse.ArgumentParser( description="Edit Jira issues locally from your filesystem", add_help=False, ) parser.add_argument("command", type=str, choices=commands.keys()) parser.add_argument("--subtasks", action="store_true", default=False) parser.add_argument( "--log-level", default="INFO", dest="log_level", ) parser.add_argument("--folder", default=os.getcwd()) parser.add_argument( "--no-subfolders", action="store_true", default=False, ) parser.add_argument( "--traceback", action="store_true", default=False, ) parser.add_argument( "--profile", action="store_true", default=False, help= "Generate a stack profile for this operation; requires 'pyinstrument'.", ) parser.add_argument( "--debugger-port", default=58024, help=("(Requires --debugger) start debugger on this port."), ) parser.add_argument( "--debugger", action="store_true", default=False, help=("Launch ptvsd debugger on --debugger-port."), ) args, extra = parser.parse_known_args() if args.debugger: try: import ptvsd ptvsd.enable_attach(address=("localhost", args.debugger_port)) print(f"{term.magenta}Waiting for debugger connection on " f"port {args.debugger_port}...{term.normal}") ptvsd.wait_for_attach() except ImportError: print( f"{term.red}Module 'ptvsd' required for debugging.{term.normal}" ) sys.exit(1) if args.profile: try: import pyinstrument profiler = pyinstrument.Profiler() profiler.start() except ImportError: print(f"{term.red}Module 'pyinstrument' required for profiling." f"{term.normal}") sys.exit(1) logging.config.dictConfig(LOGGING) root_logger = logging.getLogger() root_logger.setLevel(logging.getLevelName(args.log_level)) command_name = args.command cmd_class = commands[command_name] # Subtasks if args.subtasks: cmd_class.RUN_FOR_SUBTASKS = True started = time.time() logger.debug("Command %s(%s) started", command_name, extra) jira = utils.lazy_get_jira() try: value = cmd_class.execute_command(extra, jira=jira, path=args.folder, command_name=command_name) logger.debug( "Command %s(%s) finished in %s seconds", command_name, extra, (time.time() - started), ) if value: value.echo() sys.exit(value.return_code) except GitCommandError as e: print("{t.red}Error (code: {code}) while running git " "command.{t.normal}".format(t=term, code=e.returncode)) print("") print("{t.red}Command:{t.normal}{t.red}{t.bold}".format(t=term)) print(" {cmd}".format(cmd=e.command)) print("{t.normal}".format(t=term)) print("{t.red}Output:{t.normal}{t.red}{t.bold}".format(t=term)) for line in e.output.decode("utf8").split("\n"): print(" %s" % line) print("{t.normal}".format(t=term)) if args.traceback: traceback.print_exc() sys.exit(10) except NotTicketFolderException: if not getattr(cmd_class, "TRY_SUBFOLDERS", False): print("{t.red}The command '{cmd}' must be ran from " "within an issue folder.{t.normal}".format(t=term, cmd=command_name)) sys.exit(20) elif args.no_subfolders: sys.exit(20) count_runs = 0 for folder in os.listdir(os.getcwd()): full_path = os.path.join( os.getcwd(), folder, ) if not os.path.isdir(full_path): continue try: full_args = copy.copy(sys.argv) if "--no-subfolders" not in full_args: full_args.append("--no-subfolders") result = subprocess.call( " ".join([shlex.quote(a) for a in full_args]), cwd=full_path, shell=True, ) if result == 0: count_runs += 1 except NotTicketFolderException: pass if count_runs == 0: if args.traceback: traceback.print_exc() sys.exit(21) except UnknownMacroError as e: print("{t.red}Jirafs encountered an unknown macro while processing " "your content: {t.normal}{t.red}{t.bold}{error}" "{t.normal}".format(t=term, error=str(e))) if args.traceback: traceback.print_exc() sys.exit(30) except MacroError as e: print("{t.red}Jirafs encountered an error while processing a " "{t.bold}{macro}{t.normal}{t.red} macro: " "{t.normal}{t.red}{t.bold}{error}{t.normal}".format( t=term, macro=e.macro_name or "?", error=str(e))) if args.traceback: traceback.print_exc() sys.exit(40) except JIRAError as e: print("{t.red}Jirafs encountered an error while interacting with " "your Jira instance: {t.normal}{t.red}{t.bold}{error}" "{t.normal}".format(t=term, error=str(e))) if args.traceback: traceback.print_exc() sys.exit(70) except JiraInteractionFailed as e: print("{t.red}Jira was unable to satisfy your " "request: {t.normal}{t.red}{t.bold}{error}{t.normal}".format( t=term, error=str(e))) if args.traceback: traceback.print_exc() sys.exit(80) except JirafsError as e: print("{t.red}Jirafs encountered an error processing your " "request: {t.normal}{t.red}{t.bold}{error}{t.normal}".format( t=term, error=str(e))) if args.traceback: traceback.print_exc() sys.exit(90) finally: if args.profile: profiler.stop() print(profiler.output_text(unicode=True, color=True))