def profiled_func(*args, **kwargs):
try:
profile = pyinstrument.Profiler()
except:
profile = pyinstrument.Profiler(use_signal=False)
try:
profile.start()
result = func(*args, **kwargs)
profile.stop()
return result
finally:
print(profile.output_text(color=True))
def from_pyinstrument_trace(file_ref):
import pyinstrument
profiler = pyinstrument.Profiler()
profiler.add(file_ref)
root_frame = profiler.root_frame()
def visit_frame(frame):
code_pos = frame.code_position_short
if code_pos is None:
code_pos = ''
index = code_pos.find('site-packages')
if index > 0:
code_pos = code_pos[index + (len('site-packages') + 1):]
measurement = Measurement(
'{function} {code_pos}'.format(function=frame.function or '',
code_pos=code_pos),
self_time=datetime.timedelta(seconds=frame.self_time))
for child_frame in frame.children:
measurement.add_child(visit_frame(child_frame))
return measurement
return visit_frame(root_frame)
def time_pyinstrument(function, repeats):
timer = Timer(stmt=function)
p = pyinstrument.Profiler()
p.start()
result = timer.repeat(number=repeats)
p.stop()
return result
def main():
opt = get_opt()
if opt.dbg:
util.do_dbg = True
# just list?
if opt.list:
descriptors.list_descriptors()
return
# just check stat_data.py?
if opt.check_wt_stat_data:
descriptors.check_wt_stat_data()
return
if opt.profile:
# pip install -e git+https://github.com/joerick/pyinstrument.git#egg=pyinstrument
# pylint: disable=import-error
import pyinstrument, codecs
p = pyinstrument.Profiler()
p.start()
flow.main(opt)
p.stop()
out = codecs.getwriter('UTF-8')(sys.stderr)
out.write(p.output_text(unicode=True, color=True))
else:
flow.main(opt)
def profile_if_necessary(profiler_name, file_path):
if profiler_name == 'pyinstrument':
profiler = pyinstrument.Profiler(use_signal=False)
profiler.start()
try:
yield
finally:
if profiler_name == 'pyinstrument':
profiler.stop()
profiler.save(filename=os.path.join(file_path, 'launch-task.trace'))
def main():
p = pyinstrument.Profiler()
p.start()
t_lz4 = timeit.timeit(lambda: StoryData.decode(StoryData(
random_content(), version=StoryData.VERSION_LZ4).encode()), number=10000)
print(t_lz4)
p.stop()
html = p.output_html()
with open('benchmark_story_data_lz4.html', 'w') as f:
f.write(html)
p = pyinstrument.Profiler()
p.start()
t_gzip = timeit.timeit(lambda: StoryData.decode(StoryData(
random_content(), version=StoryData.VERSION_GZIP).encode()), number=10000)
print(t_gzip)
p.stop()
html = p.output_html()
with open('benchmark_story_data_gzip.html', 'w') as f:
f.write(html)
def start(self) -> None:
"""
Begin profiling.
"""
self.start_time = time.time()
self._profiler = cProfile.Profile()
self._callstack_profiler = pyinstrument.Profiler()
self._profiler.enable()
self._callstack_profiler.start()
def profile_solution(n_clicks: int, name: str) -> int:
if not name:
return {}
module = sys.modules[__name__]
func = getattr(module, name)
print(inspect.getsource(func))
with pi.Profiler() as profiler:
func()
profile_str = profiler.output(renderer=pi.renderers.JSONRenderer(
show_all=False))
profile_dict = json.loads(s=profile_str)
return profile_dict
def start_profiler():
import pyinstrument
profiler = pyinstrument.Profiler()
profiler.start()
return profiler
timings = g.collections.defaultdict(dict)
iterations = 10
repeats = 3
for file_name in file_names:
for test_name, test_string in tests.items():
time_min = min(timeit.repeat(test_string,
setup.format(file_name),
repeat=repeats,
number=iterations))
timings[test_name][file_name] = time_min / iterations
result = {}
result['cpu_info'] = subprocess.check_output(['cat', '/proc/cpuinfo'])
result['baseline'] = baseline.tolist()
result['timestamp'] = time.time()
result['timings'] = timings
'''
import pyinstrument
profiler = pyinstrument.Profiler()
profiler.start()
typical_application()
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
def run(self, **kwargs):
"""Main routine of the model.
Arguments:
kwargs (:obj:`dict`):
"""
for arg, val in kwargs.items():
setattr(self, arg, val)
with self.timers["setup"]:
self.setup()
logging.info("Reading restarts:")
restart.read_restart(self.itt)
diagnostics.read_restart(self)
self.enditt = self.itt + int(self.runlen / self.dt_tracer)
logging.info("Starting integration for {:.2e}s".format(self.runlen))
logging.info(" from time step {} to {}".format(self.itt,self.enditt))
try:
while self.itt < self.enditt:
if self.itt == 3 and self.profile_mode:
# when using bohrium, most kernels should be pre-compiled
# after three iterations
import pyinstrument
profiler = pyinstrument.Profiler()
profiler.start()
with self.timers["main"]:
self.set_forcing()
if self.enable_idemix:
idemix.set_idemix_parameter(self)
if self.enable_idemix_M2 or self.enable_idemix_niw:
idemix.set_spectral_parameter(self)
eke.set_eke_diffusivities(self)
tke.set_tke_diffusivities(self)
with self.timers["momentum"]:
momentum.momentum(self)
with self.timers["temperature"]:
thermodynamics.thermodynamics(self)
if self.enable_eke or self.enable_tke or self.enable_idemix:
advection.calculate_velocity_on_wgrid(self)
with self.timers["eke"]:
if self.enable_eke:
eke.integrate_eke(self)
with self.timers["idemix"]:
if self.enable_idemix_M2:
idemix.integrate_idemix_M2(self)
if self.enable_idemix_niw:
idemix.integrate_idemix_niw(self)
if self.enable_idemix:
idemix.integrate_idemix(self)
if self.enable_idemix_M2 or self.enable_idemix_niw:
idemix.wave_interaction(self)
with self.timers["tke"]:
if self.enable_tke:
tke.integrate_tke(self)
if self.enable_cyclic_x:
cyclic.setcyclic_x(self.u[:,:,:,self.taup1])
cyclic.setcyclic_x(self.v[:,:,:,self.taup1])
if self.enable_tke:
cyclic.setcyclic_x(self.tke[:,:,:,self.taup1])
if self.enable_eke:
cyclic.setcyclic_x(self.eke[:,:,:,self.taup1])
if self.enable_idemix:
cyclic.setcyclic_x(self.E_iw[:,:,:,self.taup1])
if self.enable_idemix_M2:
cyclic.setcyclic_x(self.E_M2[:,:,:,self.taup1])
if self.enable_idemix_niw:
cyclic.setcyclic_x(self.E_niw[:,:,:,self.taup1])
# diagnose vertical velocity at taup1
if self.enable_hydrostatic:
momentum.vertical_velocity(self)
self.flush()
with self.timers["diagnostics"]:
diagnostics.sanity_check(self)
if self.enable_neutral_diffusion and self.enable_skew_diffusion:
isoneutral.isoneutral_diag_streamfunction(self)
diagnostics.diagnose(self)
# shift time
otaum1 = self.taum1
self.taum1 = self.tau
self.tau = self.taup1
self.taup1 = otaum1
self.itt += 1
logging.info("Current iteration: {}".format(self.itt))
logging.debug("Time step took {}s".format(self.timers["main"].getLastTime()))
except:
diagnostics.panic_output(self)
raise
finally:
logging.debug("Timing summary:")
logging.debug(" setup time summary = {}s".format(self.timers["setup"].getTime()))
logging.debug(" main loop time summary = {}s".format(self.timers["main"].getTime()))
logging.debug(" momentum = {}s".format(self.timers["momentum"].getTime()))
logging.debug(" pressure = {}s".format(self.timers["pressure"].getTime()))
logging.debug(" friction = {}s".format(self.timers["friction"].getTime()))
logging.debug(" thermodynamics = {}s".format(self.timers["temperature"].getTime()))
logging.debug(" lateral mixing = {}s".format(self.timers["isoneutral"].getTime()))
logging.debug(" vertical mixing = {}s".format(self.timers["vmix"].getTime()))
logging.debug(" equation of state = {}s".format(self.timers["eq_of_state"].getTime()))
logging.debug(" EKE = {}s".format(self.timers["eke"].getTime()))
logging.debug(" IDEMIX = {}s".format(self.timers["idemix"].getTime()))
logging.debug(" TKE = {}s".format(self.timers["tke"].getTime()))
logging.debug(" diagnostics and I/O = {}s".format(self.timers["diagnostics"].getTime()))
if self.profile_mode:
try:
profiler.stop()
with open("profile.html", "w") as f:
f.write(profiler.output_html())
except UnboundLocalError: # profiler has not been started
pass
def test_corrected_jets_factory():
import os
from coffea.jetmet_tools import CorrectedJetsFactory, CorrectedMETFactory, JECStack
events = None
from coffea.nanoevents import NanoEventsFactory
factory = NanoEventsFactory.from_root(
os.path.abspath("tests/samples/nano_dy.root"))
events = factory.events()
jec_stack_names = [
"Summer16_23Sep2016V3_MC_L1FastJet_AK4PFPuppi",
"Summer16_23Sep2016V3_MC_L2Relative_AK4PFPuppi",
"Summer16_23Sep2016V3_MC_L2L3Residual_AK4PFPuppi",
"Summer16_23Sep2016V3_MC_L3Absolute_AK4PFPuppi",
"Spring16_25nsV10_MC_PtResolution_AK4PFPuppi",
"Spring16_25nsV10_MC_SF_AK4PFPuppi",
]
for key in evaluator.keys():
if "Summer16_23Sep2016V3_MC_UncertaintySources_AK4PFPuppi" in key:
jec_stack_names.append(key)
jec_inputs = {name: evaluator[name] for name in jec_stack_names}
jec_stack = JECStack(jec_inputs)
name_map = jec_stack.blank_name_map
name_map["JetPt"] = "pt"
name_map["JetMass"] = "mass"
name_map["JetEta"] = "eta"
name_map["JetA"] = "area"
jets = events.Jet
jets["pt_raw"] = (1 - jets["rawFactor"]) * jets["pt"]
jets["mass_raw"] = (1 - jets["rawFactor"]) * jets["mass"]
jets["pt_gen"] = ak.values_astype(ak.fill_none(jets.matched_gen.pt, 0),
np.float32)
jets["rho"] = ak.broadcast_arrays(events.fixedGridRhoFastjetAll,
jets.pt)[0]
name_map["ptGenJet"] = "pt_gen"
name_map["ptRaw"] = "pt_raw"
name_map["massRaw"] = "mass_raw"
name_map["Rho"] = "rho"
jec_cache = cachetools.Cache(np.inf)
print(name_map)
tic = time.time()
jet_factory = CorrectedJetsFactory(name_map, jec_stack)
toc = time.time()
print("setup corrected jets time =", toc - tic)
tic = time.time()
prof = pyinstrument.Profiler()
prof.start()
corrected_jets = jet_factory.build(jets, lazy_cache=jec_cache)
prof.stop()
toc = time.time()
print("corrected_jets build time =", toc - tic)
print(prof.output_text(unicode=True, color=True, show_all=True))
tic = time.time()
print("Generated jet pt:", corrected_jets.pt_gen)
print("Original jet pt:", corrected_jets.pt_orig)
print("Raw jet pt:", jets.pt_raw)
print("Corrected jet pt:", corrected_jets.pt)
print("Original jet mass:", corrected_jets.mass_orig)
print("Raw jet mass:", jets["mass_raw"])
print("Corrected jet mass:", corrected_jets.mass)
print("jet eta:", jets.eta)
for unc in jet_factory.uncertainties():
print(unc)
print(corrected_jets[unc].up.pt)
print(corrected_jets[unc].down.pt)
toc = time.time()
print("build all jet variations =", toc - tic)
# Test that the corrections were applied correctly
from coffea.jetmet_tools import (
FactorizedJetCorrector,
JetResolution,
JetResolutionScaleFactor,
)
scalar_form = ak.without_parameters(jets["pt_raw"]).layout.form
corrector = FactorizedJetCorrector(
**{name: evaluator[name]
for name in jec_stack_names[0:4]})
corrs = corrector.getCorrection(JetEta=jets["eta"],
Rho=jets["rho"],
JetPt=jets["pt_raw"],
JetA=jets["area"])
reso = JetResolution(
**{name: evaluator[name]
for name in jec_stack_names[4:5]})
jets["jet_energy_resolution"] = reso.getResolution(
JetEta=jets["eta"],
Rho=jets["rho"],
JetPt=jets["pt_raw"],
form=scalar_form,
lazy_cache=jec_cache,
)
resosf = JetResolutionScaleFactor(
**{name: evaluator[name]
for name in jec_stack_names[5:6]})
jets["jet_energy_resolution_scale_factor"] = resosf.getScaleFactor(
JetEta=jets["eta"], lazy_cache=jec_cache)
# Filter out the non-deterministic (no gen pt) jets
def smear_factor(jetPt, pt_gen, jersf):
return (ak.full_like(jetPt, 1.0) +
(jersf[:, 0] - ak.full_like(jetPt, 1.0)) *
(jetPt - pt_gen) / jetPt)
test_gen_pt = ak.concatenate(
[corrected_jets.pt_gen[0, :-2], corrected_jets.pt_gen[-1, :-1]])
test_raw_pt = ak.concatenate([jets.pt_raw[0, :-2], jets.pt_raw[-1, :-1]])
test_pt = ak.concatenate(
[corrected_jets.pt[0, :-2], corrected_jets.pt[-1, :-1]])
test_eta = ak.concatenate([jets.eta[0, :-2], jets.eta[-1, :-1]])
test_jer = ak.concatenate([
jets.jet_energy_resolution[0, :-2], jets.jet_energy_resolution[-1, :-1]
])
test_jer_sf = ak.concatenate([
jets.jet_energy_resolution_scale_factor[0, :-2],
jets.jet_energy_resolution_scale_factor[-1, :-1],
])
test_jec = ak.concatenate([corrs[0, :-2], corrs[-1, :-1]])
test_corrected_pt = ak.concatenate(
[corrected_jets.pt[0, :-2], corrected_jets.pt[-1, :-1]])
test_corr_pt = test_raw_pt * test_jec
test_pt_smear_corr = test_corr_pt * smear_factor(test_corr_pt, test_gen_pt,
test_jer_sf)
# Print the results of the "by-hand" calculations and confirm that the values match the expected values
print("\nConfirm the CorrectedJetsFactory values:")
print("Jet pt (gen)", test_gen_pt.tolist())
print("Jet pt (raw)", test_raw_pt.tolist())
print("Jet pt (nano):", test_pt.tolist())
print("Jet eta:", test_eta.tolist())
print("Jet energy resolution:", test_jer.tolist())
print("Jet energy resolution sf:", test_jer_sf.tolist())
print("Jet energy correction:", test_jec.tolist())
print("Corrected jet pt (ref)", test_corr_pt.tolist())
print("Corrected & smeared jet pt (ref):", test_pt_smear_corr.tolist())
print("Corrected & smeared jet pt:", test_corrected_pt.tolist(), "\n")
assert ak.all(np.abs(test_pt_smear_corr - test_corrected_pt) < 1e-6)
name_map["METpt"] = "pt"
name_map["METphi"] = "phi"
name_map["JetPhi"] = "phi"
name_map["UnClusteredEnergyDeltaX"] = "MetUnclustEnUpDeltaX"
name_map["UnClusteredEnergyDeltaY"] = "MetUnclustEnUpDeltaY"
tic = time.time()
met_factory = CorrectedMETFactory(name_map)
toc = time.time()
print("setup corrected MET time =", toc - tic)
met = events.MET
tic = time.time()
# prof = pyinstrument.Profiler()
# prof.start()
corrected_met = met_factory.build(met,
corrected_jets,
lazy_cache=jec_cache)
# prof.stop()
toc = time.time()
# print(prof.output_text(unicode=True, color=True, show_all=True))
print("corrected_met build time =", toc - tic)
tic = time.time()
print(corrected_met.pt_orig)
print(corrected_met.pt)
prof = pyinstrument.Profiler()
prof.start()
for unc in jet_factory.uncertainties() + met_factory.uncertainties():
print(unc)
print(corrected_met[unc].up.pt)
print(corrected_met[unc].down.pt)
prof.stop()
toc = time.time()
print("build all met variations =", toc - tic)
print(prof.output_text(unicode=True, color=True, show_all=True))
def main():
term = Terminal()
if sys.version_info < (2, 7):
raise RuntimeError(
"Jirafs requires minimally version 2.7 of Python 2, or "
"any version of Python 3. Please upgrade your version of "
"python before using Jirafs.")
if utils.get_git_version() < LooseVersion("1.8"):
raise RuntimeError(
"Jirafs requires minimally version 1.8 of Git. Please "
"upgrade your version of git before using Jirafs.")
commands = utils.get_installed_commands()
parser = argparse.ArgumentParser(
description="Edit Jira issues locally from your filesystem",
add_help=False,
)
parser.add_argument("command", type=str, choices=commands.keys())
parser.add_argument("--subtasks", action="store_true", default=False)
parser.add_argument(
"--log-level",
default="INFO",
dest="log_level",
)
parser.add_argument("--folder", default=os.getcwd())
parser.add_argument(
"--no-subfolders",
action="store_true",
default=False,
)
parser.add_argument(
"--traceback",
action="store_true",
default=False,
)
parser.add_argument(
"--profile",
action="store_true",
default=False,
help=
"Generate a stack profile for this operation; requires 'pyinstrument'.",
)
parser.add_argument(
"--debugger-port",
default=58024,
help=("(Requires --debugger) start debugger on this port."),
)
parser.add_argument(
"--debugger",
action="store_true",
default=False,
help=("Launch ptvsd debugger on --debugger-port."),
)
args, extra = parser.parse_known_args()
if args.debugger:
try:
import ptvsd
ptvsd.enable_attach(address=("localhost", args.debugger_port))
print(f"{term.magenta}Waiting for debugger connection on "
f"port {args.debugger_port}...{term.normal}")
ptvsd.wait_for_attach()
except ImportError:
print(
f"{term.red}Module 'ptvsd' required for debugging.{term.normal}"
)
sys.exit(1)
if args.profile:
try:
import pyinstrument
profiler = pyinstrument.Profiler()
profiler.start()
except ImportError:
print(f"{term.red}Module 'pyinstrument' required for profiling."
f"{term.normal}")
sys.exit(1)
logging.config.dictConfig(LOGGING)
root_logger = logging.getLogger()
root_logger.setLevel(logging.getLevelName(args.log_level))
command_name = args.command
cmd_class = commands[command_name]
# Subtasks
if args.subtasks:
cmd_class.RUN_FOR_SUBTASKS = True
started = time.time()
logger.debug("Command %s(%s) started", command_name, extra)
jira = utils.lazy_get_jira()
try:
value = cmd_class.execute_command(extra,
jira=jira,
path=args.folder,
command_name=command_name)
logger.debug(
"Command %s(%s) finished in %s seconds",
command_name,
extra,
(time.time() - started),
)
if value:
value.echo()
sys.exit(value.return_code)
except GitCommandError as e:
print("{t.red}Error (code: {code}) while running git "
"command.{t.normal}".format(t=term, code=e.returncode))
print("")
print("{t.red}Command:{t.normal}{t.red}{t.bold}".format(t=term))
print(" {cmd}".format(cmd=e.command))
print("{t.normal}".format(t=term))
print("{t.red}Output:{t.normal}{t.red}{t.bold}".format(t=term))
for line in e.output.decode("utf8").split("\n"):
print(" %s" % line)
print("{t.normal}".format(t=term))
if args.traceback:
traceback.print_exc()
sys.exit(10)
except NotTicketFolderException:
if not getattr(cmd_class, "TRY_SUBFOLDERS", False):
print("{t.red}The command '{cmd}' must be ran from "
"within an issue folder.{t.normal}".format(t=term,
cmd=command_name))
sys.exit(20)
elif args.no_subfolders:
sys.exit(20)
count_runs = 0
for folder in os.listdir(os.getcwd()):
full_path = os.path.join(
os.getcwd(),
folder,
)
if not os.path.isdir(full_path):
continue
try:
full_args = copy.copy(sys.argv)
if "--no-subfolders" not in full_args:
full_args.append("--no-subfolders")
result = subprocess.call(
" ".join([shlex.quote(a) for a in full_args]),
cwd=full_path,
shell=True,
)
if result == 0:
count_runs += 1
except NotTicketFolderException:
pass
if count_runs == 0:
if args.traceback:
traceback.print_exc()
sys.exit(21)
except UnknownMacroError as e:
print("{t.red}Jirafs encountered an unknown macro while processing "
"your content: {t.normal}{t.red}{t.bold}{error}"
"{t.normal}".format(t=term, error=str(e)))
if args.traceback:
traceback.print_exc()
sys.exit(30)
except MacroError as e:
print("{t.red}Jirafs encountered an error while processing a "
"{t.bold}{macro}{t.normal}{t.red} macro: "
"{t.normal}{t.red}{t.bold}{error}{t.normal}".format(
t=term, macro=e.macro_name or "?", error=str(e)))
if args.traceback:
traceback.print_exc()
sys.exit(40)
except JIRAError as e:
print("{t.red}Jirafs encountered an error while interacting with "
"your Jira instance: {t.normal}{t.red}{t.bold}{error}"
"{t.normal}".format(t=term, error=str(e)))
if args.traceback:
traceback.print_exc()
sys.exit(70)
except JiraInteractionFailed as e:
print("{t.red}Jira was unable to satisfy your "
"request: {t.normal}{t.red}{t.bold}{error}{t.normal}".format(
t=term, error=str(e)))
if args.traceback:
traceback.print_exc()
sys.exit(80)
except JirafsError as e:
print("{t.red}Jirafs encountered an error processing your "
"request: {t.normal}{t.red}{t.bold}{error}{t.normal}".format(
t=term, error=str(e)))
if args.traceback:
traceback.print_exc()
sys.exit(90)
finally:
if args.profile:
profiler.stop()
print(profiler.output_text(unicode=True, color=True))
