def test_basic(self):
yappi.set_clock_type('wall')
def dummy():
pass
def a():
time.sleep(0.2)
class Worker1(threading.Thread):
def a(self):
time.sleep(0.3)
def run(self):
self.a()
yappi.start(builtins=False, profile_threads=True)
c = Worker1()
c.start()
c.join()
a()
stats = yappi.get_func_stats()
fsa1 = utils.find_stat_by_name(stats, 'Worker1.a')
fsa2 = utils.find_stat_by_name(stats, 'a')
self.assertTrue(fsa1 is not None)
self.assertTrue(fsa2 is not None)
self.assertTrue(fsa1.ttot > 0.2)
self.assertTrue(fsa2.ttot > 0.1)
tstats = yappi.get_thread_stats()
self.assertEqual(len(tstats), 2)
tsa = utils.find_stat_by_name(tstats, 'Worker1')
tsm = utils.find_stat_by_name(tstats, '_MainThread')
dummy() # call dummy to force ctx name to be retrieved again.
self.assertTrue(tsa is not None)
# TODO: I put dummy() to fix below, remove the comments after a while.
self.assertTrue( # FIX: I see this fails sometimes?
tsm is not None,
'Could not find "_MainThread". Found: %s' % (', '.join(utils.get_stat_names(tstats))))
def main() -> None:
'''Runs server'''
# Parse options
define('production',
default = False,
help = 'run in production mode',
type = bool)
options.parse_command_line()
# Set server name
pname = settings.process_name if settings.process_name else None
if pname:
setproctitle(pname)
# Register IRC server
server = IRCServer(settings = ircdsettings)
for address, port in ircdsettings['listen']:
server.listen(port, address = address)
# Start profiling
if settings.profiling:
import yappi
yappi.start()
# Setup autoreload
autoreload.start()
# Run application
IOLoop.instance().start()
def main4():
import yappi
yappi.start()
main2()
stats = yappi.get_func_stats()
stats.sort('ttot')
stats.print_all()
def __init__(self, port, num_workers):
self._workers = {}
self.num_workers = num_workers
self._port = port
self._server = rpc.listen("0.0.0.0", port)
self._server.register_object(self)
self._initialized = False
self._server.serve_nonblock()
self._ctx = None
self._worker_statuses = {}
self._worker_scores = {}
self._available_workers = []
self._arrays = weakref.WeakSet()
if FLAGS.profile_master:
import yappi
yappi.start()
atexit.register(_dump_profile)
atexit.register(_shutdown)
global MASTER
MASTER = self
def test_filter(self):
def a(): pass
def b(): a()
def c(): b()
_TCOUNT = 5
ts = []
yappi.start()
for i in range(_TCOUNT):
t = threading.Thread(target=c)
t.start()
ts.append(t)
for t in ts:
t.join()
yappi.stop()
fstats = yappi.get_func_stats(filter={"ctx_id":9})
self.assertTrue(fstats.empty())
fstats = yappi.get_func_stats(filter={"ctx_id":0, "name":"c"}) # main thread
self.assertTrue(fstats.empty())
for i in range(1, _TCOUNT):
fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"a",
"ncall":1})
self.assertEqual(fstats.pop().ncall, 1)
fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"b"})
self.assertEqual(fstats.pop().ncall, 1)
fstats = yappi.get_func_stats(filter={"ctx_id":i, "name":"c"})
self.assertEqual(fstats.pop().ncall, 1)
def test_yappi_overhead(self):
import time
LOOP_COUNT = 10000
def a(): pass
def b():
for i in range(LOOP_COUNT): a()
t0 = time.time()
yappi.start()
b()
yappi.stop()
time_with_yappi = time.time() - t0
t0 = time.time()
b()
time_without_yappi = time.time() - t0
if time_without_yappi == 0:
time_without_yappi = 0.000001
# in latest v0.82, I calculated this as close to "7.0" in my machine.
# however, %83 of this overhead is coming from tickcount(). The other %17
# seems to have been evenly distributed to the internal bookkeeping
# structures/algorithms which seems acceptable. Note that our test only
# tests one function being profiled at-a-time in a short interval.
# profiling high number of functions in a small time
# is a different beast, (which is pretty unlikely in most applications)
# So as a conclusion: I cannot see any optimization window for Yappi that
# is worth implementing as we will only optimize %17 of the time.
sys.stderr.write("\r\nYappi puts %0.1f times overhead to the profiled application in average.\r\n" % \
(time_with_yappi / time_without_yappi))
def test_pause_resume(self):
yappi.set_context_id_callback(lambda: self.context_id)
yappi.set_clock_type('wall')
# Start in context 0.
self.context_id = 0
yappi.start()
time.sleep(0.08)
# Switch to context 1.
self.context_id = 1
time.sleep(0.05)
# Switch back to context 0.
self.context_id = 0
time.sleep(0.07)
yappi.stop()
t_stats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(2, len(t_stats))
self.assertEqual(0, t_stats[0].id)
self.assertEqual(2, t_stats[0].sched_count)
self.assertTrue(0.15 < t_stats[0].ttot < 0.3)
self.assertEqual(1, t_stats[1].id)
self.assertEqual(1, t_stats[1].sched_count)
# Context 1 was first scheduled 0.08 sec after context 0.
self.assertTrue(0.1 < t_stats[1].ttot < 0.2 )
def test_callback(self):
self.context_id = 0
yappi.set_context_id_callback(lambda: self.context_id)
yappi.start()
a()
self.context_id = 1
a()
self.context_id = 2
a()
# Re-schedule context 1.
self.context_id = 1
a()
yappi.stop()
threadstats = yappi.get_thread_stats().sort('id', 'ascending')
self.assertEqual(3, len(threadstats))
self.assertEqual(0, threadstats[0].id)
self.assertEqual(1, threadstats[1].id)
self.assertEqual(2, threadstats[2].id)
self.assertEqual(1, threadstats[0].sched_count)
self.assertEqual(2, threadstats[1].sched_count) # Context 1 ran twice.
self.assertEqual(1, threadstats[2].sched_count)
funcstats = yappi.get_func_stats()
self.assertEqual(4, utils.find_stat_by_name(funcstats, 'a').ncall)
def test_profile_single_context(self):
def id_callback():
return self.callback_count
def a():
pass
self.callback_count = 1
yappi.set_context_id_callback(id_callback)
yappi.start(profile_threads=False)
a() # context-id:1
self.callback_count = 2
a() # context-id:2
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, "a")
self.assertEqual(fsa.ncall, 1)
yappi.stop()
yappi.clear_stats()
self.callback_count = 1
yappi.start() # profile_threads=True
a() # context-id:1
self.callback_count = 2
a() # context-id:2
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, "a")
self.assertEqual(fsa.ncall, 2)
def test_print_formatting(self):
def a():
pass
def b():
a()
func_cols = {1: ("name", 48), 0: ("ncall", 5), 2: ("tsub", 8)}
thread_cols = {1: ("name", 48), 0: ("ttot", 8)}
yappi.start()
a()
b()
yappi.stop()
fs = yappi.get_func_stats()
cs = fs[1].children
ts = yappi.get_thread_stats()
# fs.print_all(out=sys.stderr, columns={1:("name", 70), })
# cs.print_all(out=sys.stderr, columns=func_cols)
# ts.print_all(out=sys.stderr, columns=thread_cols)
# cs.print_all(out=sys.stderr, columns={})
self.assertRaises(yappi.YappiError, fs.print_all, columns={1: ("namee", 9)})
self.assertRaises(yappi.YappiError, cs.print_all, columns={1: ("dd", 0)})
self.assertRaises(yappi.YappiError, ts.print_all, columns={1: ("tidd", 0)})
def test_singlethread_profiling(self):
yappi.set_clock_type("wall")
def a():
time.sleep(0.2)
class Worker1(threading.Thread):
def a(self):
time.sleep(0.3)
def run(self):
self.a()
yappi.start(profile_threads=False)
c = Worker1()
c.start()
c.join()
a()
stats = yappi.get_func_stats()
fsa1 = utils.find_stat_by_name(stats, "Worker1.a")
fsa2 = utils.find_stat_by_name(stats, "a")
self.assertTrue(fsa1 is None)
self.assertTrue(fsa2 is not None)
self.assertTrue(fsa2.ttot > 0.1)
def test_merge_multithreaded_stats(self):
import threading
import _yappi
timings = {"a_1":2, "b_1":1}
_yappi._set_test_timings(timings)
def a(): pass
def b(): pass
yappi.start()
t = threading.Thread(target=a)
t.start()
t.join()
t = threading.Thread(target=b)
t.start()
t.join()
yappi.get_func_stats().save("ystats1.ys")
yappi.clear_stats()
_yappi._set_test_timings(timings)
self.assertEqual(len(yappi.get_func_stats()), 0)
self.assertEqual(len(yappi.get_thread_stats()), 1)
t = threading.Thread(target=a)
t.start()
t.join()
self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0)
self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1)
yappi.get_func_stats().save("ystats2.ys")
stats = yappi.YFuncStats(["ystats1.ys", "ystats2.ys",])
fsa = utils.find_stat_by_name(stats, "a")
fsb = utils.find_stat_by_name(stats, "b")
self.assertEqual(fsa.ncall, 2)
self.assertEqual(fsb.ncall, 1)
self.assertEqual(fsa.tsub, fsa.ttot, 4)
self.assertEqual(fsb.tsub, fsb.ttot, 1)
def test_children_stat_functions(self):
_timings = {"a_1":5, "b_1":3, "c_1":1}
_yappi._set_test_timings(_timings)
def b():
pass
def c():
pass
def a():
b()
c()
yappi.start()
a()
b() # non-child call
c() # non-child call
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, 'a')
childs_of_a = fsa.children.get().sort("tavg", "desc")
prev_item = None
for item in childs_of_a:
if prev_item:
self.assertTrue(prev_item.tavg > item.tavg)
prev_item = item
childs_of_a.sort("name", "desc")
prev_item = None
for item in childs_of_a:
if prev_item:
self.assertTrue(prev_item.name > item.name)
prev_item = item
childs_of_a.clear()
self.assertTrue(childs_of_a.empty())
def test_basic(self):
import threading
import time
yappi.set_clock_type('wall')
def a():
time.sleep(0.2)
class Worker1(threading.Thread):
def a(self):
time.sleep(0.3)
def run(self):
self.a()
yappi.start(builtins=False, profile_threads=True)
c = Worker1()
c.start()
c.join()
a()
stats = yappi.get_func_stats()
fsa1 = utils.find_stat_by_name(stats, 'Worker1.a')
fsa2 = utils.find_stat_by_name(stats, 'a')
self.assertTrue(fsa1 is not None)
self.assertTrue(fsa2 is not None)
self.assertTrue(fsa1.ttot > 0.2)
self.assertTrue(fsa2.ttot > 0.1)
tstats = yappi.get_thread_stats()
self.assertEqual(len(tstats), 2)
tsa = utils.find_stat_by_name(tstats, 'Worker1')
tsm = utils.find_stat_by_name(tstats, '_MainThread')
self.assertTrue(tsa is not None)
self.assertTrue(tsm is not None) # FIX: I see this fails sometimes?
def start(self):
yappi.set_context_id_callback(lambda:id(greenlet.getcurrent()))
yappi.set_context_name_callback(lambda:greenlet.getcurrent().__class__.__name__)
yappi.set_clock_type('cpu')
yappi.start(builtins=True)
self._isStart = True
return 'success'
def test_producer_consumer_with_queues(self):
# we currently just stress yappi, no functionality test is done here.
yappi.start()
import time
if utils.is_py3x():
from queue import Queue
else:
from Queue import Queue
from threading import Thread
WORKER_THREAD_COUNT = 50
WORK_ITEM_COUNT = 2000
def worker():
while True:
item = q.get()
# do the work with item
q.task_done()
q = Queue()
for i in range(WORKER_THREAD_COUNT):
t = Thread(target=worker)
t.daemon = True
t.start()
for item in range(WORK_ITEM_COUNT):
q.put(item)
q.join()# block until all tasks are done
#yappi.get_func_stats().sort("callcount").print_all()
yappi.stop()
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--host', default=None, # NOT -h, that's help
help='host that coordinated will listen on, '
'0.0.0.0 for any input interface')
ap.add_argument('--port', '-p', type=int, default=None,
help='port number that coordinated will listen on')
ap.add_argument('--pid', default=None,
help='file to write pid to')
ap.add_argument('--snapshot-dir', default=None,
help='direcotry to write snapshots to')
ap.add_argument('--httpd', default=None,
help='ip:port or :port to serve http info on')
if yappi is not None:
ap.add_argument('--yappi', default=None, help='file to write yappi profiling to. will be suffied by {timestamp}.txt')
args = yakonfig.parse_args(ap, [yakonfig, dblogger, coordinate])
if args.pid:
with open(args.pid, 'w') as f:
f.write(str(os.getpid()))
if args.snapshot_dir is not None:
cjqconfig = yakonfig.get_global_config('coordinate', 'job_queue')
# (This modifies the global configuration in place)
cjqconfig['snapshot_path_format'] = os.path.join(
args.snapshot_dir, 'snapshot_{timestamp}')
if (yappi is not None) and args.yappi:
yappi.start()
yt = threading.Thread(target=yappi_logger, args=(args.yappi,))
yt.daemon = True
yt.start()
daemon = CoordinateServer(host=args.host, port=args.port, httpd=args.httpd)
daemon.run()
def test_concurrent_futures(self):
yappi.start()
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=5) as executor:
f = executor.submit(pow, 5, 2)
self.assertEqual(f.result(), 25)
time.sleep(1.0)
yappi.stop()
def wrapped(*args, **kwargs):
yappi.start()
result = func(*args, **kwargs)
yappi.stop()
prof_file = "%s.%s" % (func.__name__, time.time())
#prof_file = "callgrind.a.1"
yappi.get_func_stats().save(prof_file, "ystat")
return result
def test_lambda(self):
f = lambda: time.sleep(0.3)
yappi.set_clock_type("wall")
yappi.start()
f()
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, "<lambda>")
self.assertTrue(fsa.ttot > 0.1)
def get(self):
self.interval = int(self.get_argument('interval', self.interval))
self.sort_type = yappi.SORT_TYPES_FUNCSTATS[self.get_argument(
'sort_type', 'tsub')]
self.limit = self.get_argument('limit', self.limit)
yappi.start()
tornado.ioloop.IOLoop.instance().add_timeout(
datetime.timedelta(seconds=self.interval), self.finish_profile)
def test_start_flags(self):
self.assertEqual(_yappi._get_start_flags(), None)
yappi.start()
def a(): pass
a()
self.assertEqual(_yappi._get_start_flags()["profile_builtins"], 0)
self.assertEqual(_yappi._get_start_flags()["profile_multithread"], 1)
self.assertEqual(len(yappi.get_thread_stats()), 1)
def __init__(self, master):
# Reseed the Numpy random number state.
#
# The default initialization results in all worker processes on the machine having the
# same random seed.
np.random.seed(seed=os.getpid())
self.id = -1
self._initialized = False
self._peers = {}
self._blobs = {}
self._master = master
self._running = True
self._ctx = None
self.worker_status = core.WorkerStatus(
psutil.TOTAL_PHYMEM,
psutil.NUM_CPUS,
psutil.virtual_memory().percent,
psutil.cpu_percent(),
time.time(),
[],
[],
)
self._lock = rlock.FastRLock()
# self._lock = threading.Lock()
# Patch to fix buggy assumption by multiprocessing library
if not hasattr(threading.current_thread(), "_children"):
threading.current_thread()._children = weakref.WeakKeyDictionary()
self._kernel_threads = ThreadPool(processes=1)
self._kernel_remain_tiles = []
if FLAGS.profile_worker:
import yappi
yappi.start()
hostname = socket.gethostname()
self._server = rpc.listen_on_random_port(hostname)
self._server.register_object(self)
if FLAGS.profile_worker:
self._server._socket._event_loop.enable_profiling()
self._server.serve_nonblock()
req = core.RegisterReq()
req.host = hostname
req.port = self._server.addr[1]
req.worker_status = self.worker_status
with _init_lock:
# There is a race-condition in the initialization code for zeromq; this causes
# sporadic crashes when running in multi-thread mode. We lock the first
# client RPC to workaround this issue.
master.register(req)
def test_generator(self):
def _gen(n):
while(n > 0):
yield n
n -= 1
yappi.start()
for x in _gen(5):
pass
self.assertTrue(yappi.convert2pstats(yappi.get_func_stats()) is not None)
def start():
pages.require("/admin/settings.edit", noautoreturn=True)
pages.postOnly()
import yappi
if not yappi.is_running():
yappi.start()
time.sleep(0.5)
messagebus.postMessage("/system/settings/activatedprofiler",pages.getAcessingUser())
raise cherrypy.HTTPRedirect("/settings/profiler")
def _start_profiling(self, clock):
with lock:
if yappi.is_running():
raise http.Error(
http.BAD_REQUEST, "profile is already running")
log.info("Starting profiling using %r clock", clock)
yappi.set_clock_type(clock)
yappi.start(builtins=True, profile_threads=True)
def test_clear_stats_while_running(self):
def a():
pass
yappi.start()
a()
yappi.clear_stats()
a()
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, 'a')
self.assertEqual(fsa.ncall, 1)
def yappi_prof_call(func, *args):
'''
https://code.google.com/p/yappi/wiki/usageyappi_v092
'''
import yappi
yappi.start()
result = func(*args)
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
return result
def start_cpu_profiling():
"""
If the environment variable W3AF_PROFILING is set to 1, then we start
the CPU and memory profiling.
:return: None
"""
import yappi
yappi.start()
dump_data_every_thread(dump_data, DELAY_MINUTES, SAVE_THREAD_PTR)
def profile():
"""
Code profiler using YAPPI
http://code.google.com/p/yappi
"""
import yappi
yappi.start()
start(False)
yappi.stop()
for pstat in yappi.get_stats(yappi.SORTTYPE_TSUB):
print pstat
def main(args, datasets):
do_prof = args.do_profile
do_tensorflow = not args.disable_tensorflow
#use the environment variable for cupy/cuda choice
args.use_cuda = USE_CUPY
analysis_corrections = None
if "analyze" in args.action:
analysis_corrections = AnalysisCorrections(args, do_tensorflow)
# Optionally disable pinned memory (will be somewhat slower)
if args.use_cuda:
import cupy
if not args.pinned:
cupy.cuda.set_allocator(None)
cupy.cuda.set_pinned_memory_allocator(None)
#Use sync-only datasets
if args.do_sync:
datasets = datasets_sync
#Filter datasets by era
datasets_to_process = []
for ds in datasets:
if args.datasets is None or ds[0] in args.datasets:
if args.eras is None or ds[1] in args.eras:
datasets_to_process += [ds]
print("Will consider dataset", ds)
if len(datasets) == 0:
raise Exception("No datasets considered, please check the --datasets and --eras options")
datasets = datasets_to_process
hmumu_utils.NUMPY_LIB, hmumu_utils.ha = choose_backend(args.use_cuda)
Dataset.numpy_lib = hmumu_utils.NUMPY_LIB
NUMPY_LIB = hmumu_utils.NUMPY_LIB
# All analysis definitions (cut values etc) should go here
analysis_parameters = {
"baseline": {
"nPV": 0,
"NdfPV": 4,
"zPV": 24,
# Will be applied with OR
"hlt_bits": {
"2016": ["HLT_IsoMu24", "HLT_IsoTkMu24"],
"2017": ["HLT_IsoMu27"],
"2018": ["HLT_IsoMu24"],
},
"muon_pt": 20,
"muon_pt_leading": {"2016": 26.0, "2017": 29.0, "2018": 26.0},
"muon_eta": 2.4,
"muon_iso": 0.25,
"muon_id": {"2016": "medium", "2017": "medium", "2018": "medium"},
"muon_trigger_match_dr": 0.1,
"muon_iso_trigger_matched": 0.15,
"muon_id_trigger_matched": {"2016": "tight", "2017": "tight", "2018": "tight"},
"do_rochester_corrections": True,
"do_lepton_sf": True,
"do_jec": True,
"jec_tag": {"2016": "Summer16_07Aug2017_V11", "2017": "Fall17_17Nov2017_V32", "2018": "Autumn18_V16"},
"jet_mu_dr": 0.4,
"jet_pt_leading": {"2016": 35.0, "2017": 35.0, "2018": 35.0},
"jet_pt_subleading": {"2016": 25.0, "2017": 25.0, "2018": 25.0},
"jet_eta": 4.7,
"jet_id": "tight",
"jet_puid": "loose",
"jet_veto_eta": [2.65, 3.139],
"jet_veto_raw_pt": 50.0,
"jet_btag": {"2016": 0.6321, "2017": 0.4941, "2018": 0.4184},
"do_factorized_jec": args.do_factorized_jec,
"cat5_dijet_inv_mass": 400.0,
"cat5_abs_jj_deta_cut": 2.5,
"masswindow_z_peak": [76, 106],
"masswindow_h_sideband": [110, 150],
"masswindow_h_peak": [115, 135],
"inv_mass_bins": 41,
"extra_electrons_pt": 20,
"extra_electrons_eta": 2.5,
"extra_electrons_iso": 0.4, #Check if we want to apply this
"extra_electrons_id": "mvaFall17V1Iso_WP90",
"save_dnn_vars": True,
"dnn_vars_path": "{0}/dnn_vars".format(args.out),
#If true, apply mjj > cut, otherwise inverse
"vbf_filter_mjj_cut": 350,
"vbf_filter": {
"dy_m105_160_mg": True,
"dy_m105_160_amc": True,
"dy_m105_160_vbf_mg": False,
"dy_m105_160_vbf_amc": False,
},
#Irene's DNN input variable order for keras
"dnn_varlist_order": ['softJet5', 'dRmm','dEtamm','M_jj','pt_jj','eta_jj','phi_jj','M_mmjj','eta_mmjj','phi_mmjj','dEta_jj','Zep','dRmin_mj', 'dRmax_mj', 'dRmin_mmj','dRmax_mmj','dPhimm','leadingJet_pt','subleadingJet_pt', 'leadingJet_eta','subleadingJet_eta','leadingJet_qgl','subleadingJet_qgl','cthetaCS','Higgs_pt','Higgs_eta','Higgs_mass'],
"dnn_input_histogram_bins": {
"softJet5": (0,10,10),
"dRmm": (0,5,41),
"dEtamm": (-2,2,41),
"dPhimm": (-2,2,41),
"M_jj": (0,2000,41),
"pt_jj": (0,400,41),
"eta_jj": (-5,5,41),
"phi_jj": (-5,5,41),
"M_mmjj": (0,2000,41),
"eta_mmjj": (-3,3,41),
"phi_mmjj": (-3,3,41),
"dEta_jj": (-3,3,41),
"Zep": (-2,2,41),
"dRmin_mj": (0,5,41),
"dRmax_mj": (0,5,41),
"dRmin_mmj": (0,5,41),
"dRmax_mmj": (0,5,41),
"leadingJet_pt": (0, 200, 41),
"subleadingJet_pt": (0, 200, 41),
"leadingJet_eta": (-5, 5, 41),
"subleadingJet_eta": (-5, 5, 41),
"leadingJet_qgl": (0, 1, 41),
"subleadingJet_qgl": (0, 1, 41),
"cthetaCS": (-1, 1, 41),
"Higgs_pt": (0, 200, 41),
"Higgs_eta": (-3, 3, 41),
"Higgs_mass": (110, 150, 41),
"dnn_pred": (0, 1, 1001),
"dnn_pred2": (0, 1, 11),
"bdt_ucsd": (-1, 1, 41),
"bdt2j_ucsd": (-1, 1, 41),
"bdt01j_ucsd": (-1, 1, 41),
"MET_pt": (0, 200, 41),
"hmmthetacs": (-1, 1, 41),
"hmmphics": (-4, 4, 41),
},
"categorization_trees": {}
},
}
histo_bins = {
"muon_pt": np.linspace(0, 200, 101, dtype=np.float32),
"npvs": np.linspace(0,100,101, dtype=np.float32),
"dijet_inv_mass": np.linspace(0, 2000, 41, dtype=np.float32),
"inv_mass": np.linspace(70, 150, 41, dtype=np.float32),
"numjet": np.linspace(0, 10, 11, dtype=np.float32),
"jet_pt": np.linspace(0, 300, 101, dtype=np.float32),
"jet_eta": np.linspace(-4.7, 4.7, 41, dtype=np.float32),
"pt_balance": np.linspace(0, 5, 41, dtype=np.float32),
"numjets": np.linspace(0, 10, 11, dtype=np.float32),
"jet_qgl": np.linspace(0, 1, 41, dtype=np.float32),
"higgs_inv_mass_uncertainty": np.linspace(0, 10, 101, dtype=np.float32),
"higgs_rel_inv_mass_uncertainty": np.linspace(0, 0.05, 101, dtype=np.float32)
}
for hname, bins in analysis_parameters["baseline"]["dnn_input_histogram_bins"].items():
histo_bins[hname] = np.linspace(bins[0], bins[1], bins[2], dtype=np.float32)
for masswindow in ["z_peak", "h_peak", "h_sideband"]:
mw = analysis_parameters["baseline"]["masswindow_" + masswindow]
histo_bins["inv_mass_{0}".format(masswindow)] = np.linspace(mw[0], mw[1], 41, dtype=np.float32)
histo_bins["dnn_pred2"] = {
"h_peak": np.array([0., 0.9, 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 1.0], dtype=np.float32),
"z_peak": np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 1.0], dtype=np.float32),
"h_sideband": np.array([0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.0], dtype=np.float32),
}
analysis_parameters["baseline"]["histo_bins"] = histo_bins
#analysis_parameters["oldjec"] = copy.deepcopy(analysis_parameters["baseline"])
#analysis_parameters["oldjec"]["jec_tag"]["2018"] = "Autumn18_V8"
#Run baseline analysis
outpath = "{0}/partial_results".format(args.out)
try:
os.makedirs(outpath)
except FileExistsError as e:
pass
with open('{0}/parameters.pkl'.format(outpath), 'wb') as handle:
pickle.dump(analysis_parameters, handle, protocol=pickle.HIGHEST_PROTOCOL)
#Recreate dump of all filenames
cache_filename = args.cache_location + "/datasets.json"
if ("cache" in args.action) and (args.jobfiles is None):
print("--action cache and no jobfiles specified, creating datasets.json dump of all filenames")
if not os.path.isdir(args.cache_location):
os.makedirs(args.cache_location)
filenames_cache = {}
for dataset in datasets:
dataset_name, dataset_era, dataset_globpattern, is_mc = dataset
filenames_all = glob.glob(args.datapath + dataset_globpattern, recursive=True)
filenames_all = [fn for fn in filenames_all if not "Friend" in fn]
filenames_cache[dataset_name + "_" + dataset_era] = [
fn.replace(args.datapath, "") for fn in filenames_all]
if len(filenames_all) == 0:
raise Exception("Dataset {0} matched 0 files from glob pattern {1}, verify that the data files are located in {2}".format(
dataset_name, dataset_globpattern, args.datapath
))
#save all dataset filenames to a json file
print("Creating a json dump of all the dataset filenames based on data found in {0}".format(args.datapath))
if os.path.isfile(cache_filename):
print("Cache file {0} already exists, we will not overwrite it to be safe.".format(cache_filename), file=sys.stderr)
print("Delete it or change --cache-location and try again.", file=sys.stderr)
sys.exit(1)
with open(cache_filename, "w") as fi:
fi.write(json.dumps(filenames_cache, indent=2))
if ("cache" in args.action or "analyze" in args.action) and (args.jobfiles is None):
#Create a list of job files for processing
jobfile_data = []
print("Loading list of filenames from {0}".format(cache_filename))
if not os.path.isfile(cache_filename):
raise Exception("Cached dataset list of filenames not found in {0}, please run this code with --action cache".format(
cache_filename))
filenames_cache = json.load(open(cache_filename, "r"))
for dataset in datasets:
dataset_name, dataset_era, dataset_globpattern, is_mc = dataset
try:
filenames_all = filenames_cache[dataset_name + "_" + dataset_era]
except KeyError as e:
print("Could not load {0} from {1}, please make sure this dataset has been added to cache".format(
dataset_name + "_" + dataset_era, cache_filename), file=sys.stderr)
raise e
filenames_all_full = [args.datapath + "/" + fn for fn in filenames_all]
chunksize = args.chunksize * chunksize_multiplier.get(dataset_name, 1)
print("Saving dataset {0}_{1} with {2} files in {3} files per chunk to jobfiles".format(
dataset_name, dataset_era, len(filenames_all_full), chunksize))
jobfile_dataset = create_dataset_jobfiles(dataset_name, dataset_era,
filenames_all_full, is_mc, chunksize, args.out)
jobfile_data += jobfile_dataset
print("Dataset {0}_{1} consists of {2} chunks".format(
dataset_name, dataset_era, len(jobfile_dataset)))
assert(len(jobfile_data) > 0)
assert(len(jobfile_data[0]["filenames"]) > 0)
#For each dataset, find out which chunks we want to process
if "cache" in args.action or "analyze" in args.action:
jobfile_data = []
if not (args.jobfiles_load is None):
args.jobfiles = [l.strip() for l in open(args.jobfiles_load).readlines()]
if args.jobfiles is None:
print("You did not specify to process specific dataset chunks, assuming you want to process all chunks")
print("If this is not true, please specify e.g. --jobfiles data_2018_0.json data_2018_1.json ...")
args.jobfiles = []
for dataset in datasets:
dataset_name, dataset_era, dataset_globpattern, is_mc = dataset
jobfiles_dataset = glob.glob(args.out + "/jobfiles/{0}_{1}_*.json".format(dataset_name, dataset_era))
assert(len(jobfiles_dataset) > 0)
if args.maxchunks > 0:
jobfiles_dataset = jobfiles_dataset[:args.maxchunks]
args.jobfiles += jobfiles_dataset
#Now load the jobfiles
assert(len(args.jobfiles) > 0)
print("You specified --jobfiles {0}, processing only these dataset chunks".format(" ".join(args.jobfiles)))
jobfile_data = []
for f in args.jobfiles:
jobfile_data += [json.load(open(f))]
chunkstr = " ".join(["{0}_{1}_{2}".format(
ch["dataset_name"], ch["dataset_era"], ch["dataset_num_chunk"])
for ch in jobfile_data])
print("Will process {0} dataset chunks: {1}".format(len(jobfile_data), chunkstr))
assert(len(jobfile_data) > 0)
#Start the profiler only in the actual data processing
if do_prof:
import yappi
filename = 'analysis.prof'
yappi.set_clock_type('cpu')
yappi.start(builtins=True)
if "cache" in args.action:
print("Running the 'cache' step of the analysis, ROOT files will be opened and branches will be uncompressed")
print("Will retrieve dataset filenames based on existing ROOT files on filesystem in datapath={0}".format(args.datapath))
try:
os.makedirs(cmdline_args.cache_location)
except Exception as e:
pass
run_cache(args, outpath, jobfile_data, analysis_parameters)
if "analyze" in args.action:
run_analysis(args, outpath, jobfile_data, analysis_parameters, analysis_corrections)
if "merge" in args.action:
with ProcessPoolExecutor(max_workers=args.nthreads) as executor:
for dataset in datasets:
dataset_name, dataset_era, dataset_globpattern, is_mc = dataset
fut = executor.submit(merge_partial_results, dataset_name, dataset_era, outpath)
print("done merging")
if do_prof:
stats = yappi.get_func_stats()
stats.save(filename, type='callgrind')
import resource
total_memory = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
total_memory += resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print("maxrss={0} MB".format(total_memory/1024))
def setup_profiler(self):
if not self.is_profiling():
return
yappi.start()
def injection_load(**kwargs):
import yappi
if not yappi.is_running():
yappi.start()
def __init__(self):
yappi.start()
self.timer = threading.Timer(self.TRACING_INTERVAL,
self._logTracingStats)
self.timer.start()
def main(args):
do_prof = args.do_profile
do_tensorflow = not args.disable_tensorflow
# use the environment variable for cupy/cuda choice
args.use_cuda = USE_CUPY
datasets = yaml.load(open(args.datasets_yaml),
Loader=yaml.FullLoader)["datasets"]
# Filter datasets by era
datasets_to_process = []
for ds in datasets:
if args.datasets is None or ds["name"] in args.datasets:
if args.eras is None or ds["era"] in args.eras:
datasets_to_process += [ds]
if len(datasets_to_process) == 0:
raise Exception(
"No datasets considered, please check the --datasets and --eras options"
)
datasets = datasets_to_process
# Choose either the CPU or GPU(CUDA) backend
hmumu_utils.NUMPY_LIB, hmumu_utils.ha = choose_backend(args.use_cuda)
Dataset.numpy_lib = hmumu_utils.NUMPY_LIB
outpath_partial = "{0}/partial_results".format(args.out)
try:
os.makedirs(outpath_partial)
except FileExistsError:
print("Output path {0} already exists, not recreating".format(
outpath_partial))
# save the parameters as a pkl file
from pars import analysis_parameters
for analysis_name in analysis_parameters.keys():
analysis_parameters[analysis_name][
"do_factorized_jec"] = args.do_factorized_jec
analysis_parameters[analysis_name][
"dnn_vars_path"] = "{0}/dnn_vars".format(args.out)
with open('{0}/parameters.pkl'.format(outpath_partial), 'wb') as handle:
pickle.dump(analysis_parameters,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
# Recreate dump of all filenames
cache_filename = "{0}/datasets.json".format(args.out)
use_skim = False
if args.cachepath is None:
print(
"--cachepath not specified, will process unskimmed NanoAOD, which is somewhat slower!"
)
print("Please see the README.md on how to skim the NanoAOD")
datapath = args.datapath
else:
print("Processing skimmed NanoAOD")
datapath = args.cachepath
use_skim = True
check_and_recreate_filename_cache(cache_filename, datapath, datasets,
use_skim)
# Create the jobfiles
if args.jobfiles is None:
create_all_jobfiles(datasets, cache_filename, datapath, args.chunksize,
args.out)
# For each dataset, find out which chunks we want to process
if "analyze" in args.action:
jobfile_data = load_jobfiles(datasets, args.jobfiles_load,
args.jobfiles, args.maxchunks, args.out)
# If we want to check what part of the code is slow, start the profiler only in the actual data processing
if do_prof:
import yappi
yappi.set_clock_type('cpu')
yappi.start(builtins=True)
# Run the physics analysis on all specified jobfiles
if "analyze" in args.action:
print(
"Running the 'analyze' step of the analysis, processing the events into histograms with all systematics"
)
analysis_corrections = AnalysisCorrections(args, do_tensorflow)
run_analysis(args, outpath_partial, jobfile_data, analysis_parameters,
analysis_corrections)
if do_prof:
stats = yappi.get_func_stats()
stats.save("analysis.prof", type='callgrind')
# Merge the partial results (pieces of each dataset)
if "merge" in args.action:
with ProcessPoolExecutor(max_workers=args.nthreads) as executor:
for dataset in datasets:
dataset_name = dataset["name"]
dataset_era = dataset["era"]
executor.submit(merge_partial_results, dataset_name,
dataset_era, args.out, outpath_partial)
print("done merging")
# print memory usage for debugging
total_memory = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
total_memory += resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print("maxrss={0} MB".format(total_memory / 1024))
def run_with_yappi(func, *args, **kwargs):
yappi.start()
func(*args, **kwargs)
yappi.stop()
from plugin_manager import *
from enthought.traits.api import HasTraits, Str, Regex, List, Instance, DelegatesTo
from enthought.traits.ui.api import TreeEditor, TreeNode, View, Item, VSplit, \
HGroup, Controller, Handler, Group, Include, ValueEditor, HSplit, ListEditor, InstanceEditor
from enthought.traits.ui.menu import Menu, Action, Separator, MenuBar
import yaml
import wx
PROFILE = False
PROFILE_BUILTINS = True
if PROFILE:
import yappi
yappi.start(PROFILE_BUILTINS)
def open_file(wildcard='',
dir='./saved_sessions',
file_name='',
message='Open'):
dialog = wx.FileDialog(None,
wildcard=wildcard,
defaultFile=file_name,
defaultDir=dir,
message=message,
style=wx.OPEN | wx.FILE_MUST_EXIST)
file_name = ''
if dialog.ShowModal() == wx.ID_OK:
file_name = dialog.GetPath()
def setUp(self):
self.prof_start = prof.start()
yappi.start(builtins=False)
def test_basic(self):
@asyncio.coroutine
def a():
yield from async_sleep(0.3)
burn_cpu(0.4)
@asyncio.coroutine
def b():
yield from a()
@asyncio.coroutine
def recursive_a(n):
if not n:
return
burn_io(0.3)
yield from async_sleep(0.3)
yield from recursive_a(n - 1)
tlocal = threading.local()
def tag_cbk():
try:
return tlocal._tag
except:
return -1
yappi.set_clock_type("wall")
tlocal._tag = 0
yappi.set_tag_callback(tag_cbk)
def _thread_event_loop(loop, tag):
tlocal._tag = tag
asyncio.set_event_loop(loop)
loop.run_forever()
_TCOUNT = 3
_ctag = 1
ts = []
for i in range(_TCOUNT):
_loop = asyncio.new_event_loop()
t = threading.Thread(target=_thread_event_loop,
args=(_loop, _ctag))
t._loop = _loop
t.start()
ts.append(t)
_ctag += 1
@asyncio.coroutine
def stop_loop():
asyncio.get_event_loop().stop()
@asyncio.coroutine
def driver():
futs = []
fut = asyncio.run_coroutine_threadsafe(a(), ts[0]._loop)
futs.append(fut)
fut = asyncio.run_coroutine_threadsafe(recursive_a(5), ts[1]._loop)
futs.append(fut)
fut = asyncio.run_coroutine_threadsafe(b(), ts[2]._loop)
futs.append(fut)
for fut in futs:
fut.result()
# stop asyncio loops in threads
for t in ts:
asyncio.run_coroutine_threadsafe(stop_loop(), t._loop)
yappi.start()
asyncio.get_event_loop().run_until_complete(driver())
yappi.stop()
traces = yappi.get_func_stats()
t1 = '''
tests/test_asyncio.py:137 driver 1 0.000061 3.744064 3.744064
tests/test_asyncio.py:96 recursive_a 6/1 0.000188 3.739663 0.623277
tests/test_asyncio.py:8 async_sleep 7 0.000085 2.375271 0.339324
tests/utils.py:135 burn_io 5 0.000044 1.700000 0.437400
tests/test_asyncio.py:87 a 2 0.000019 1.600000 0.921138
tests/utils.py:126 burn_cpu 2 0.800000 0.800000 0.509730
tests/test_asyncio.py:92 b 1 0.000005 0.800000 0.921055
'''
self.assert_traces_almost_equal(t1, traces)
traces = yappi.get_func_stats(filter={'tag': 2})
t1 = '''
tests/test_asyncio.py:96 recursive_a 6/1 0.000211 3.720011 0.620002
tests/utils.py:135 burn_io 5 0.000079 1.700000 0.431813
async_sleep 5 0.000170 1.560735 0.312147
'''
self.assert_traces_almost_equal(t1, traces)
def test_profile_decorator(self):
def aggregate(func, stats):
fname = "%s.profile" % (func.__name__)
try:
stats.add(fname)
except IOError:
pass
stats.save(fname)
raise Exception("messing around")
@yappi.profile(return_callback=aggregate)
def a(x, y):
if x + y == 25:
raise Exception("")
return x + y
def b():
pass
try:
os.remove(
"a.profile") # remove the one from prev test, if available
except:
pass
# global profile is on to mess things up
yappi.start()
b()
# assert functionality and call function at same time
try:
self.assertEqual(a(1, 2), 3)
except:
pass
try:
self.assertEqual(a(2, 5), 7)
except:
pass
try:
a(4, 21)
except:
pass
stats = yappi.get_func_stats().add("a.profile")
fsa = utils.find_stat_by_name(stats, 'a')
self.assertEqual(fsa.ncall, 3)
self.assertEqual(len(stats), 1) # b() should be cleared out.
@yappi.profile(return_callback=aggregate)
def count_down_rec(n):
if n == 0:
return
count_down_rec(n - 1)
try:
os.remove("count_down_rec.profile"
) # remove the one from prev test, if available
except:
pass
try:
count_down_rec(4)
except:
pass
try:
count_down_rec(3)
except:
pass
stats = yappi.YFuncStats("count_down_rec.profile")
fsrec = utils.find_stat_by_name(stats, 'count_down_rec')
self.assertEqual(fsrec.ncall, 9)
self.assertEqual(fsrec.nactualcall, 2)
if opts.aircraft is not None:
start_script = os.path.join(opts.aircraft, "mavinit.scr")
if os.path.exists(start_script):
run_script(start_script)
else:
print("no script %s" % start_script)
if opts.cmd is not None:
for cstr in opts.cmd:
cmds = cstr.split(';')
for c in cmds:
process_stdin(c)
if opts.profile:
import yappi # We do the import here so that we won't barf if run normally and yappi not available
yappi.start()
# log all packets from the master, for later replay
open_telemetry_logs(logpath_telem, logpath_telem_raw)
# run main loop as a thread
mpstate.status.thread = threading.Thread(target=main_loop, name='main_loop')
mpstate.status.thread.daemon = True
mpstate.status.thread.start()
# use main program for input. This ensures the terminal cleans
# up on exit
while (mpstate.status.exit != True):
try:
if opts.daemon:
time.sleep(0.1)
async def profile_tasks_coroutine(self):
yappi.start()
tasks = self.get_tasks()
await asyncio.gather(*tasks)
yappi.stop()
def start(self):
yappi.start()
import yappi
def estat(e):
print e
def foo():
pass
yappi.start(True)
for i in xrange(100000):
foo()
yappi.stop()
#yappi.enum_stats(estat)
yappi.print_stats()
yappi.clear_stats()
import yappi
import threading
class deadlock_thread(threading.Thread):
# acquire the lock two times to create a very
# simple deadlock:)
def foo(self):
for i in xrange(0, 2):
self.lock.acquire()
def run(self):
self.lock = threading.Lock()
self.foo()
yappi.start() # start the profiler
thr = deadlock_thread() # start the deadlock thread
thr.start()
while 1:
s = raw_input("yappi>")
exec s
def enable(self):
if self.wall:
yappi.clear_stats()
yappi.set_clock_type("wall")
yappi.start()
def main(args):
"""Main entry point allowing external calls
Args:
args ([str]): command line parameter list
"""
# Create $HOME/tmp directory if it does not exist
(Path.home() / "tmp").mkdir(exist_ok=True)
# Define logger format and handlers
logger = logging.getLogger("transfer_vn")
# create file handler which logs even debug messages
fh = TimedRotatingFileHandler(
str(Path.home()) + "/tmp/transfer_vn.log",
when="midnight",
interval=1,
backupCount=100,
)
# create console handler with a higher log level
ch = logging.StreamHandler()
# create formatter and add it to the handlers
formatter = logging.Formatter(
"%(asctime)s - %(levelname)s - %(name)s - %(message)s")
fh.setFormatter(formatter)
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
logger.addHandler(ch)
# Get command line arguments
args = arguments(args)
# Start profiling if required
if args.profile:
yappi.start()
logger.info(_("Started yappi"))
# Define verbosity
if args.verbose:
logger.setLevel(logging.DEBUG)
sql_quiet = ""
client_min_message = "debug1"
else:
logger.setLevel(logging.INFO)
sql_quiet = "--quiet"
client_min_message = "warning"
logger.info(_("%s, version %s"), sys.argv[0], __version__)
logger.info(_("Arguments: %s"), sys.argv[1:])
# If required, first create YAML file
if args.init:
logger.info(_("Creating YAML configuration file"))
init(args.file)
return None
# Get configuration from file
if not (Path.home() / args.file).is_file():
logger.critical(_("File %s does not exist"),
str(Path.home() / args.file))
return None
logger.info(_("Getting configuration data from %s"), args.file)
try:
cfg_ctrl = EvnConf(args.file)
except YAMLValidationError:
logger.critical(_("Incorrect content in YAML configuration %s"),
args.file)
sys.exit(0)
cfg_site_list = cfg_ctrl.site_list
cfg = list(cfg_site_list.values())[0]
# Check configuration consistency
if cfg.db_enabled and cfg.json_format != "short":
logger.critical(
_("Storing to Postgresql cannot use long json_format."))
logger.critical(_("Please modify YAML configuration and restart."))
sys.exit(0)
manage_pg = PostgresqlUtils(cfg)
if args.db_drop:
logger.info(_("Delete if exists database and roles"))
manage_pg.drop_database()
if args.db_create:
logger.info(_("Create database and roles"))
manage_pg.create_database()
if args.json_tables_create:
logger.info(_("Create, if not exists, json tables"))
manage_pg.create_json_tables()
if args.col_tables_create:
logger.info(_("Creating or recreating vn columns based tables"))
col_table_create(cfg, sql_quiet, client_min_message)
if args.full:
logger.info(_("Performing a full download"))
full_download(cfg_ctrl)
if args.schedule:
logger.info(_("Creating or modifying incremental download schedule"))
increment_schedule(cfg_ctrl)
if args.update:
logger.info(_("Performing an incremental download"))
increment_download(cfg_ctrl)
if args.status:
logger.info(_("Printing download status"))
status(cfg_ctrl)
if args.count:
logger.info(_("Counting observations"))
count_observations(cfg_ctrl)
# Stop and output profiling if required
if args.profile:
logger.info(_("Printing yappi results"))
yappi.stop()
yappi.get_func_stats().print_all()
yappi.get_thread_stats().print_all()
return None
def clusterArgInicial(idtese):
#Variaveis e funçoes para conexação com o banco de dados do Debate de Teses
cursor = connection.cursor()
cursor2 = connection.cursor()
cursor.execute(
"select distinct `usr`.`primeironome` as `name`, `arg`.`argumento` AS `posicionamentoinicial` from ((((`argumento` `arg` join `revisao` `rev`) join `replica` `rep`) join `posicionamento` `pos`) join `argumentador` `urg`)join `usuario` `usr` where ((`arg`.`tese_idtese` = "
+ idtese +
" ) and (`rev`.`argumento_idargumento` = `arg`.`idargumento`) and (`rep`.`revisao_idrevisao` = `rev`.`idrevisao`) and (`arg`.`argumentador_idargumentador` = `pos`.`argumentador_idargumentador`) and (`arg`.`tese_idtese` = `pos`.`tese_idtese`) and (`arg`.`posicionamentoinicial` is not null) and (`arg`.`argumentador_idargumentador` = `urg`.`idargumentador`) and(`urg`.`usuario_idusuario` = `usr`.`idusuario`) and (`pos`.`posicionamentofinal` is not null))"
)
cursor2.execute("select tese from tese where idtese=" + idtese)
#Variavel e função para tratar tags html e acentos com codificação ISO
h = HTMLParser.HTMLParser()
#dados retirados da consulta ao banco
dadosSql = cursor.fetchall()
textotese = cursor2.fetchall()
#listas para tratar os dados iniciais
usu = []
posInicial = []
dados = []
tese = []
#lista com dados pos tagger
tag_posInicial = []
tag_comAce_posInicial = []
#lista com dados após a remoção das stopwords
sw_tese = []
sw_posInicial = []
aux_usu = []
sw_tagPosInicial = [] #texto marcado e sem stopwords
sw_tagcomAce_posInicial = [] #texto COM ACENTOS marcado e sem stopwords
#lista com dados após a aplicação de Stemming
st_posInicial = []
st_tese = []
st_tagPosInicial = [] #texto marcado, sem stopwords e com stemmer aplicado
st_tagcomAce_posInicial = [
] #texto COM ACENTOS marcado, sem stopwords e com stemmer aplicado
#############################################################################################################
#LISTA COM OS POSICIONAMENTOS INICIAIS APÓS APLICAÇÃO DA NORMALIZAÇAÕ
posInicial_Normalizado = []
normalizacao = []
#############################################################################################################
#Aplicacao de Case Folding
for d in dadosSql:
dados.append([
re.sub('<[^>]*>', '', h.unescape(d[0])).lower(),
re.sub('<[^>]*>', '', h.unescape(d[1])).lower()
])
for t in textotese:
tese.append(re.sub('<[^>]*>', '', h.unescape(t[0])).lower())
#Colocando os textos de posicionamento inicial em numa lista separada
for i in dados:
x = 0
usu.append(i[x].upper())
posInicial.append(
i[x + 1].lower()
) #lista com o posicionamento Inicial com todas as letras em minusculo
#############################################################################################################
### Classificacao das palavras de acordo com sua classe gramatical
### Utilizacao do postagger NLPNET
### http://nilc.icmc.usp.br/nlpnet/index.html#
tagger = nlpnet.POSTagger()
semAce_posInicial = [
] #armazena o posInicial apenas sem acentos, sem pontuações, sem endereço web e sem numeros
comAce_posInicial = [
] #armazena o posInicial apenas COM acentos, sem pontuações, sem endereço web e sem numeros
for i in posInicial:
semAce_posInicial.append(
removePontuacao(removeA(removeNum(removeSE(removeEndWeb((i)))))))
for i in semAce_posInicial:
tag_posInicial.append(tagger.tag(i))
for i in posInicial:
comAce_posInicial.append(
removePontuacao(removeNum(removeSE(removeEndWeb((i))))))
for i in comAce_posInicial:
tag_comAce_posInicial.append(tagger.tag(i))
#############################################################################################################
#APENAS PARA REALIZAR TESTE E COLOCAR NA DISSERTACAO
# pprint(semAce_posInicial)
# pprint(comAce_posInicial)
# exit()
# tagg_posInicial = []
# for texto in posInicial:
# tagg_posInicial.append(tagger.tag(texto))
#
# print "posInicial"
# pprint(posInicial)
#
# print "tagg_posInicial"
# pprint(tagg_posInicial)
#############################################################################################################
#############################################################################################################
### REMOCAO DE STOPWORDS
### Remocao dos termos de acordo com a NLTK
### Remocao dos termos classificados como artigos, verbos, adverbios, etc...
for i in usu:
aux_usu.append(removeStopWords(i))
for i in tese:
sw_tese.append(removeStopWords(i))
for i in posInicial:
sw_posInicial.append(removeStopWords(i))
for i in tag_posInicial:
sw_tagPosInicial.append(limpaCorpus(i))
for i in tag_comAce_posInicial:
sw_tagcomAce_posInicial.append(limpaCorpus(i))
####################################################################################################################################
# Aplicação do RSPL Stemmer para remoção dos afixos das palavras da lingua portuguesa
# Retirando afixos dos textos do posInicial e tese
stemmer = RSLPStemmer()
for i in range(len(sw_posInicial)):
st_aux = sw_posInicial[i]
string_aux = ""
for sufixo in st_aux.split():
string_aux = string_aux + " " + stemmer.stem(sufixo)
st_posInicial.append(string_aux)
for i in range(len(sw_tese)):
st_aux = sw_tese[i]
string_aux = ""
for sufixo in st_aux.split():
string_aux = string_aux + " " + stemmer.stem(sufixo)
st_tese.append(string_aux)
for i in range(len(sw_tagPosInicial)):
termosST = ""
auxST = []
for j in range(len(sw_tagPosInicial[i])):
aux = stemmer.stem(sw_tagPosInicial[i][j][0])
etiqueta = sw_tagPosInicial[i][j][1]
termosST = (aux, etiqueta)
auxST.append(termosST)
st_tagPosInicial.append(auxST)
for i in range(len(sw_tagcomAce_posInicial)):
termosST = ""
auxST = []
for j in range(len(sw_tagcomAce_posInicial[i])):
aux = stemmer.stem(sw_tagcomAce_posInicial[i][j][0])
etiqueta = sw_tagcomAce_posInicial[i][j][1]
termosST = (aux, etiqueta)
auxST.append(termosST)
st_tagcomAce_posInicial.append(auxST)
####################################################################################################################################
### A NORMALIZACAO DE TERMOS REFERE-SE A TECNICA DE TROCAR PALAVRAS SINONIMAS, OU SEJA, QUE TENHAM SIGNIFICADO ##
### SEMELHANTE, POR UM UNICO TERMO REPRESENTATIVO NO CORPUS DE ANALISE. DESSA FORMA, É POSSIVEL AUMENTAR O GRAU ##
### DE SIMILARIDADE ENTRE OS TEXTOS ANALISADOS ATRAVES DO USO DE TECNICAS DE ANALISE ESTATISTICAS, COMO SIMILA ##
### RIDADE DE COSSENOS OU DISTANCIA EUCLIDIANA. ##
####################################################################################################################################
### A NORMALIZACAO FOI DESENVOLVIDA COM BASE NOS DADOS DISPONIBILIZADOS PELO PROJETO TEP 2.0 DO NILC/USP ##
### http://143.107.183.175:21480/tep2/index.htm ##
### ##
### FORMATO DO ARQUIVO ##
### NUM1. [Tipo] {termos sinonimos} <NUM2> ##
### 263. [Verbo] {consentir, deixar, permitir} <973> ##
### NUM1 = NUMERO DA LINHA DE REFERENCIA PARA TERMO SINONIMO ##
### NUM2 = NUMERO DA LINHA DE REFERENCIA PARA TERMO ANTONIMO (SENTIDO OPOSTO) ##
####################################################################################################################################
#abre o arquivo com as relacoes de sinonimia (termos linhaWordNet) e antonimia (termos contrarios)
#arquivo apenas com termos classificados como substantivos, adjetivos e verbos
base_tep = codecs.open(
os.path.join(os.path.dirname(__file__), '../base_tep2/base_tep.txt'),
'r', 'UTF8')
# dicionario = open('/home/panceri/git/alpes_v1/base_tep2/dicionarioSinonimos.txt', 'w')
#variavel com conteúdo do arquivo em memoria
#não imprimir essa variável, MUITO GRANDEE!!!
wordNet = base_tep.readlines()
#fechar arquivo
base_tep.close()
####################################################################################################################################
## NORMALIZAÇÃO FEITA COM BASE NOS RADICAIS DE FORMAÇÃO DAS PALAVRAS ##
## APLICAÇÃO DO RSPL PRIMEIRO PARA DEPOIS BUSCAR NA BASE OS TERMOS SIMILARES ##
## DENTRO DA BASE_TEP OS TERMOS TAMBÉM FORAM REDUZIDOS AOS SEUS RADICIAIS DE FORMAÇÃO ##
## O DICIONÁRIO ESTÁ COM A REFERÊNCIA PARA A LINHA AONDE ESTÃO OS TERMOS SINÔNIMOS ##
## OS TERMOS SÃO ANALISADOS CONSIDERANDO SUAS ACENTUAÇÕES, PARA APLICAÇÃO CORRETA DO RSLP ##
####################################################################################################################################
yappi.set_clock_type('cpu')
yappi.start(builtins=True)
start = time.time()
st_WordNetV = [
] ##armazena num, tipo, e radical dos sinonimos - APENAS VERBOS
st_WordNetN = [
] ##armazena num, tipo, e radical dos sinonimos - APENAS SUBSTANTIVOS
st_WordNetA = [
] ##armazena num, tipo, e radical dos sinonimos - APENAS ADJETIVOS
st_WordNetO = [
] ##armazena num, tipo, e radical dos sinonimos - APENAS OUTROS
for linhaWordnet in wordNet:
listaAux = []
termos = re.findall(r"\{(.*)\}", linhaWordnet)
num = re.findall(r"([0-9]+)\.", linhaWordnet)
tipo = re.findall(r"\[(.*)\]", linhaWordnet)
if tipo[0] == "Substantivo":
listaAux.append(num)
listaAux.append(tipo)
for T in termos:
aux = T.split()
auxL = []
for i in aux:
aux1 = i.replace(",", "")
dadosStem = stemmer.stem(aux1)
auxL.append(dadosStem)
listaAux.append(auxL)
st_WordNetN.append(listaAux)
elif tipo[0] == "Verbo":
listaAux.append(num)
listaAux.append(tipo)
for T in termos:
aux = T.split()
auxL = []
for i in aux:
aux1 = i.replace(",", "")
dadosStem = stemmer.stem(aux1)
auxL.append(dadosStem)
listaAux.append(auxL)
st_WordNetV.append(listaAux)
elif tipo[0] == "Adjetivo":
listaAux.append(num)
listaAux.append(tipo)
for T in termos:
aux = T.split()
auxL = []
for i in aux:
aux1 = i.replace(",", "")
dadosStem = stemmer.stem(aux1)
auxL.append(dadosStem)
listaAux.append(auxL)
st_WordNetA.append(listaAux)
else:
listaAux.append(num)
listaAux.append(tipo)
for T in termos:
aux = T.split()
auxL = []
for i in aux:
aux1 = i.replace(",", "")
dadosStem = stemmer.stem(aux1)
auxL.append(dadosStem)
listaAux.append(auxL)
st_WordNetO.append(listaAux)
duration = time.time() - start
stats = yappi.get_func_stats()
stats.save('stemmWordNet.out', type='callgrind')
####################################################################################################################################
### A ANÁLISE É REALIZADA COM BASE NO TEXTO SEM A EXCLUSÃO DOS ACENTOS ##
### POIS AO EXCLUÍ-LOS A REDUÇÃO AO RADICAL DE FORMAÇÃO (APLICAÇÃO DO RSLP) É PREJUDICADA ##
### OS TESTES REALIZADOS MOSTRARAM QUE ESSA É UMA MELHOR ABORDAGEM, UMA VEZ QUE NOSSOS TEXTOS SÃO PEQUENOS ##
### E PRECISAMOS CHEGAR O MAIS PRÓXIMO POSSÍVEL SEM CONSIDERAR SEUS SENTIDOS E/OU CONTEXTOS ##
####################################################################################################################################
yappi.set_clock_type('cpu')
yappi.start(builtins=True)
start = time.time()
normalizacao = normalizacaoWordnet(st_WordNetA, st_WordNetN, st_WordNetV,
st_WordNetO, st_tagcomAce_posInicial)
###############################################################
# Colocando os textos normalizados numa lista de 1 diemensão
###############################################################
stringNorm = ""
auxNorm = []
for i in range(len(normalizacao)):
auxNorm = normalizacao[i]
for x in range(len(auxNorm)):
stringNorm = stringNorm + " " + auxNorm[x]
posInicial_Normalizado.append(stringNorm)
stringNorm = ""
duration = time.time() - start
stats = yappi.get_func_stats()
stats.save('normalizacaoWordnet.out', type='callgrind')
####################################################################################################################################
# print "posInicial"
# pprint(posInicial)
#
# print "comAce_posInicial"
# pprint(comAce_posInicial)
#
# print "tag_comAce_posInicial"
# pprint(tag_comAce_posInicial)
#
# print "sw_tagcomAce_posInicial"
# pprint(sw_tagcomAce_posInicial)
#
# print "st_tagcomAce_posInicial"
# pprint(st_tagcomAce_posInicial)
# print "posInicial_Normalizado"
# print len(posInicial_Normalizado)
# pprint(posInicial_Normalizado)
# exit()
####################################################################################################################################
#retorno da função - usado na views.py para alimentar o template debate.html
#passar parametros que devem ser apresentados na templates debate.html
return [
st_tese, posInicial, sw_tese, aux_usu, st_posInicial, tese,
posInicial_Normalizado
]
def Apriori(self):
tStart = time.time()
candidate_num, frequnt_num, all_frequent_itemsets = self.Find_frequent_one(
self.sup)
print('L1====================================')
print('Frequent 1-itemset is', frequnt_num)
process = psutil.Process(os.getpid())
print('Used Memory:', process.memory_info().rss / 1024 / 1024, 'MB')
prev_frequent_num = frequnt_num
print('====================================')
tEnd = time.time()
print(tEnd - tStart)
prev_frequent = [x for x in all_frequent_itemsets]
prev_frequent.sort(key=lambda tup: tup)
length = 2
while len(prev_frequent) > 1:
tStart = time.time()
if self.debug:
# profilling
yappi.clear_stats() # clear profiler
yappi.set_clock_type('cpu')
yappi.start(builtins=True) # track builtins
new_candidate = self.Generate_candidate(prev_frequent, length)
print('L%s====================================' % (length))
print('Total candidates is ',
len(new_candidate) + prev_frequent_num)
self.htree = self.Generate_hash_tree(new_candidate)
self.Frequent_support(length)
# find frequent itemsets
new_frequent = self.htree.get_frequent_itemsets(self.sup)
print('Frequent itemset is', len(new_frequent) + prev_frequent_num)
process = psutil.Process(os.getpid())
print('Used Memory:',
process.memory_info().rss / 1024 / 1024, 'MB')
print('====================================')
tEnd = time.time()
print(tEnd - tStart)
prev_frequent_num = len(new_frequent) + prev_frequent_num
# all_frequent_itemsets.extend(new_frequent)
prev_frequent = [tup[0] for tup in new_frequent]
prev_frequent.sort()
if self.debug:
yappi.stop()
stat = yappi.get_func_stats()
var = 'callgrind.generate_tree' + str(length)
stat.save(var, type='callgrind')
# profilling end
length += 1
'''
gc
'''
del self.htree
del new_candidate
del new_frequent
gc.collect()
return all_frequent_itemsets
def __init__(self, args=None):
QtCore.QObject.__init__(self)
self.app = QtGui.QApplication(["FireSim"])
self.args = args
self.config = Config("data/config.json")
if self.args.profile:
try:
import yappi
yappi.start()
except ImportError:
log.error("Could not enable YaPPI profiling")
self._selected_fixture_strand = 0
self._selected_fixture_address = 0
self._selected_fixture_pixels = 0
self.selected_fixture = None
self.is_blurred = False
self.scene = Scene(
os.path.join(self.config.get("scene_root"), self.args.scene) +
".json")
self.scenecontroller = SceneController(app=self, scene=self.scene)
QtDeclarative.qmlRegisterType(CanvasWidget, "FireSim", 1, 0,
"SimCanvas")
QtDeclarative.qmlRegisterType(FixtureWidget, "FireSim", 1, 0,
"Fixture")
self.view = QtDeclarative.QDeclarativeView()
self.view.setWindowTitle("FireSim")
self.view.setResizeMode(
QtDeclarative.QDeclarativeView.SizeRootObjectToView)
self.view.closeEvent = self.on_close
self.context = self.view.rootContext()
self.context.setContextProperty('main', self)
self.fixture_info_list = []
self.context.setContextProperty('fixtureInfoModel',
self.fixture_info_list)
self.view.setSource(QtCore.QUrl('ui/qml/FireSimGUI.qml'))
self.root = self.view.rootObject()
self.item_frame = self.root.findChild(QtDeclarative.QDeclarativeItem)
self.canvas = self.root.findChild(CanvasWidget)
self.canvas.gui = self
cw, ch = self.scenecontroller.scene.extents()
self.canvas.setWidth(cw)
self.canvas.setHeight(ch)
self.scenecontroller.set_canvas(self.canvas)
self.root.backdrop_showhide_callback.connect(
self.on_btn_backdrop_showhide)
self.root.labels_showhide_callback.connect(self.on_btn_labels_showhide)
self.root.lock_callback.connect(self.on_btn_lock)
self.root.show_center_callback.connect(self.on_btn_show_center)
self.root.toggle_blurred_callback.connect(self.on_btn_toggle_blurred)
#self.net_thread = QtCore.QThread()
#self.net_thread.start()
self.netcontroller = NetController(self)
#self.netcontroller.moveToThread(self.net_thread)
#self.netcontroller.start.emit()
self.net_stats_timer = QtCore.QTimer()
self.net_stats_timer.setInterval(1000)
self.net_stats_timer.timeout.connect(self.update_net_stats)
self.net_stats_timer.start()
self.netcontroller.data_received.connect(self.on_network_event)
self.scenecontroller.new_frame.connect(
self.netcontroller.frame_complete)
self.netcontroller.data_received.connect(
self.scenecontroller.process_command)
self.view.setFixedSize(max(640, cw + 130), max(480, ch))
log.info("FireSimGUI Ready.")
self.view.show()
def test_merge_stats(self):
_timings = {
"a_1": 15,
"b_1": 14,
"c_1": 12,
"d_1": 10,
"e_1": 9,
"f_1": 7,
"g_1": 6,
"h_1": 5,
"i_1": 1
}
_yappi._set_test_timings(_timings)
def a():
b()
def b():
c()
def c():
d()
def d():
e()
def e():
f()
def f():
g()
def g():
h()
def h():
i()
def i():
pass
yappi.start()
a()
a()
yappi.stop()
stats = yappi.get_func_stats()
self.assertRaises(NotImplementedError, stats.save, "",
"INVALID_SAVE_TYPE")
stats.save("ystats2.ys")
yappi.clear_stats()
_yappi._set_test_timings(_timings)
yappi.start()
a()
stats = yappi.get_func_stats().add("ystats2.ys")
fsa = utils.find_stat_by_name(stats, "a")
fsb = utils.find_stat_by_name(stats, "b")
fsc = utils.find_stat_by_name(stats, "c")
fsd = utils.find_stat_by_name(stats, "d")
fse = utils.find_stat_by_name(stats, "e")
fsf = utils.find_stat_by_name(stats, "f")
fsg = utils.find_stat_by_name(stats, "g")
fsh = utils.find_stat_by_name(stats, "h")
fsi = utils.find_stat_by_name(stats, "i")
self.assertEqual(fsa.ttot, 45)
self.assertEqual(fsa.ncall, 3)
self.assertEqual(fsa.nactualcall, 3)
self.assertEqual(fsa.tsub, 3)
self.assertEqual(fsa.children[fsb].ttot, fsb.ttot)
self.assertEqual(fsa.children[fsb].tsub, fsb.tsub)
self.assertEqual(fsb.children[fsc].ttot, fsc.ttot)
self.assertEqual(fsb.children[fsc].tsub, fsc.tsub)
self.assertEqual(fsc.tsub, 6)
self.assertEqual(fsc.children[fsd].ttot, fsd.ttot)
self.assertEqual(fsc.children[fsd].tsub, fsd.tsub)
self.assertEqual(fsd.children[fse].ttot, fse.ttot)
self.assertEqual(fsd.children[fse].tsub, fse.tsub)
self.assertEqual(fse.children[fsf].ttot, fsf.ttot)
self.assertEqual(fse.children[fsf].tsub, fsf.tsub)
self.assertEqual(fsf.children[fsg].ttot, fsg.ttot)
self.assertEqual(fsf.children[fsg].tsub, fsg.tsub)
self.assertEqual(fsg.ttot, 18)
self.assertEqual(fsg.tsub, 3)
self.assertEqual(fsg.children[fsh].ttot, fsh.ttot)
self.assertEqual(fsg.children[fsh].tsub, fsh.tsub)
self.assertEqual(fsh.ttot, 15)
self.assertEqual(fsh.tsub, 12)
self.assertEqual(fsh.tavg, 5)
self.assertEqual(fsh.children[fsi].ttot, fsi.ttot)
self.assertEqual(fsh.children[fsi].tsub, fsi.tsub)
def start(self):
yappi.start()
return '<a href="stats">stats</a>'
def test_merge_aabab_aabbc(self):
_timings = {
"a_1": 15,
"a_2": 14,
"b_1": 12,
"a_3": 10,
"b_2": 9,
"c_1": 4
}
_yappi._set_test_timings(_timings)
def a():
if self._ncall == 1:
self._ncall += 1
a()
elif self._ncall == 5:
self._ncall += 1
a()
else:
b()
def b():
if self._ncall == 2:
self._ncall += 1
a()
elif self._ncall == 6:
self._ncall += 1
b()
elif self._ncall == 7:
c()
else:
return
def c():
pass
self._ncall = 1
stats = utils.run_and_get_func_stats(a, )
stats.save("ystats1.ys")
yappi.clear_stats()
_yappi._set_test_timings(_timings)
#stats.print_all()
self._ncall = 5
stats = utils.run_and_get_func_stats(a, )
stats.save("ystats2.ys")
#stats.print_all()
def a(): # same name but another function(code object)
pass
yappi.start()
a()
stats = yappi.get_func_stats().add(["ystats1.ys", "ystats2.ys"])
#stats.print_all()
self.assertEqual(len(stats), 4)
fsa = None
for stat in stats:
if stat.name == "a" and stat.ttot == 45:
fsa = stat
break
self.assertTrue(fsa is not None)
self.assertEqual(fsa.ncall, 7)
self.assertEqual(fsa.nactualcall, 3)
self.assertEqual(fsa.ttot, 45)
self.assertEqual(fsa.tsub, 10)
fsb = utils.find_stat_by_name(stats, "b")
fsc = utils.find_stat_by_name(stats, "c")
self.assertEqual(fsb.ncall, 6)
self.assertEqual(fsb.nactualcall, 3)
self.assertEqual(fsb.ttot, 36)
self.assertEqual(fsb.tsub, 27)
self.assertEqual(fsb.tavg, 6)
self.assertEqual(fsc.ttot, 8)
self.assertEqual(fsc.tsub, 8)
self.assertEqual(fsc.tavg, 4)
self.assertEqual(fsc.nactualcall, fsc.ncall, 2)
def test_subsequent_profile(self):
WORKER_COUNT = 5
def a():
pass
def b():
pass
def c():
pass
_timings = {
"a_1": 3,
"b_1": 2,
"c_1": 1,
}
yappi.start()
def g():
pass
g()
yappi.stop()
yappi.clear_stats()
_yappi._set_test_timings(_timings)
yappi.start()
_dummy = []
for i in range(WORKER_COUNT):
t = threading.Thread(target=a)
t.start()
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=b)
t.start()
_dummy.append(t)
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=a)
t.start()
t.join()
for i in range(WORKER_COUNT):
t = threading.Thread(target=c)
t.start()
t.join()
yappi.stop()
yappi.start()
def f():
pass
f()
stats = yappi.get_func_stats()
fsa = utils.find_stat_by_name(stats, 'a')
fsb = utils.find_stat_by_name(stats, 'b')
fsc = utils.find_stat_by_name(stats, 'c')
self.assertEqual(fsa.ncall, 10)
self.assertEqual(fsb.ncall, 5)
self.assertEqual(fsc.ncall, 5)
self.assertEqual(fsa.ttot, fsa.tsub, 30)
self.assertEqual(fsb.ttot, fsb.tsub, 10)
self.assertEqual(fsc.ttot, fsc.tsub, 5)
# MACOSx optimizes by only creating one worker thread
self.assertTrue(len(yappi.get_thread_stats()) >= 2)
def test_ctx_stats(self):
from threading import Thread
DUMMY_WORKER_COUNT = 5
yappi.start()
class DummyThread(Thread):
pass
def dummy():
pass
def dummy_worker():
pass
for i in range(DUMMY_WORKER_COUNT):
t = DummyThread(target=dummy_worker)
t.start()
t.join()
yappi.stop()
stats = yappi.get_thread_stats()
tsa = utils.find_stat_by_name(stats, "DummyThread")
self.assertTrue(tsa is not None)
yappi.clear_stats()
time.sleep(1.0)
_timings = {
"a_1": 6,
"b_1": 5,
"c_1": 3,
"d_1": 1,
"a_2": 4,
"b_2": 3,
"c_2": 2,
"d_2": 1
}
_yappi._set_test_timings(_timings)
class Thread1(Thread):
pass
class Thread2(Thread):
pass
def a():
b()
def b():
c()
def c():
d()
def d():
time.sleep(0.6)
yappi.set_clock_type("wall")
yappi.start()
t1 = Thread1(target=a)
t1.start()
t2 = Thread2(target=a)
t2.start()
t1.join()
t2.join()
stats = yappi.get_thread_stats()
# the fist clear_stats clears the context table?
tsa = utils.find_stat_by_name(stats, "DummyThread")
self.assertTrue(tsa is None)
tst1 = utils.find_stat_by_name(stats, "Thread1")
tst2 = utils.find_stat_by_name(stats, "Thread2")
tsmain = utils.find_stat_by_name(stats, "_MainThread")
dummy() # call dummy to force ctx name to be retrieved again.
self.assertTrue(len(stats) == 3)
self.assertTrue(tst1 is not None)
self.assertTrue(tst2 is not None)
# TODO: I put dummy() to fix below, remove the comments after a while.
self.assertTrue( # FIX: I see this fails sometimes
tsmain is not None, 'Could not find "_MainThread". Found: %s' %
(', '.join(utils.get_stat_names(stats))))
self.assertTrue(1.0 > tst2.ttot >= 0.5)
self.assertTrue(1.0 > tst1.ttot >= 0.5)
# test sorting of the ctx stats
stats = stats.sort("totaltime", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.ttot >= stat.ttot)
prev_stat = stat
stats = stats.sort("totaltime", "asc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.ttot <= stat.ttot)
prev_stat = stat
stats = stats.sort("schedcount", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.sched_count >= stat.sched_count)
prev_stat = stat
stats = stats.sort("name", "desc")
prev_stat = None
for stat in stats:
if prev_stat:
self.assertTrue(prev_stat.name >= stat.name)
prev_stat = stat
self.assertRaises(yappi.YappiError, stats.sort,
"invalid_thread_sorttype_arg")
self.assertRaises(yappi.YappiError, stats.sort,
"invalid_thread_sortorder_arg")
def _on_keyboard_down(self, _keyboard, keycode, _text, modifiers):
self.last_key_down = keycode
ctrl_pressed = "ctrl" in modifiers
if self.controls.note.focus:
return # when making notes, don't allow keyboard shortcuts
popup = self.popup_open
if popup:
if keycode[1] in ["f5", "f6", "f7", "f8"]: # switch between popups
popup.dismiss()
return
elif keycode[1] in ["enter", "numpadenter"]:
fn = getattr(popup.content, "on_submit", None)
if fn:
fn()
return
else:
return
shift_pressed = "shift" in modifiers
shortcuts = self.shortcuts
if keycode[1] == "spacebar":
self.toggle_continuous_analysis()
elif keycode[1] == "k":
self.board_gui.toggle_coordinates()
elif keycode[1] in ["pause", "break", "f15"] and not ctrl_pressed:
self.controls.timer.paused = not self.controls.timer.paused
elif keycode[1] in ["`", "~", "f12"]:
self.zen = (self.zen + 1) % 3
elif keycode[1] in ["left", "z"]:
self("undo", 1 + shift_pressed * 9 + ctrl_pressed * 9999)
elif keycode[1] in ["right", "x"]:
self("redo", 1 + shift_pressed * 9 + ctrl_pressed * 9999)
elif keycode[1] == "home":
self("undo", 9999)
elif keycode[1] == "end":
self("redo", 9999)
elif keycode[1] == "pageup":
self.controls.move_tree.make_selected_node_main_branch()
elif keycode[1] == "n" and not ctrl_pressed:
self("find-mistake", "undo" if shift_pressed else "redo")
elif keycode[1] == "delete" and ctrl_pressed:
self.controls.move_tree.delete_selected_node()
elif keycode[1] == "c" and not ctrl_pressed:
self.controls.move_tree.toggle_selected_node_collapse()
elif keycode[1] == "n" and ctrl_pressed:
self("new-game-popup")
elif keycode[1] == "l" and ctrl_pressed:
self("analyze-sgf-popup")
elif keycode[1] == "s" and ctrl_pressed:
self("save-game")
elif keycode[1] == "d" and ctrl_pressed:
self("save-game-as-popup")
elif keycode[1] == "c" and ctrl_pressed:
Clipboard.copy(self.game.root.sgf())
self.controls.set_status(i18n._("Copied SGF to clipboard."), STATUS_INFO)
elif keycode[1] == "v" and ctrl_pressed:
self.load_sgf_from_clipboard()
elif keycode[1] == "b" and shift_pressed:
self("undo", "main-branch")
elif keycode[1] in shortcuts.keys() and not ctrl_pressed:
shortcut = shortcuts[keycode[1]]
if isinstance(shortcut, Widget):
shortcut.trigger_action(duration=0)
else:
self(*shortcut)
elif keycode[1] == "f9" and self.debug_level >= OUTPUT_EXTRA_DEBUG:
import yappi
yappi.set_clock_type("cpu")
yappi.start()
self.log("starting profiler", OUTPUT_ERROR)
elif keycode[1] == "f10" and self.debug_level >= OUTPUT_EXTRA_DEBUG:
import time
import yappi
stats = yappi.get_func_stats()
filename = f"callgrind.{int(time.time())}.prof"
stats.save(filename, type="callgrind")
self.log(f"wrote profiling results to {filename}", OUTPUT_ERROR)
def profile(config_location="/etc/redeem"):
import yappi
yappi.start()
main(config_location)
yappi.get_func_stats().print_all()
def filter_opencv(im):
for i in range(100000):
im += 1
return im
def filter_homemade(im):
for i in range(100000):
im += 2
return im
import yappi
filename = 'callgrind.filename.prof'
yappi.set_clock_type('cpu')
yappi.start(builtins=True)
filter_homemade(0)
filter_opencv(0)
stats = yappi.get_func_stats()
stats.save(filename, type='callgrind')
def profile():
import yappi
yappi.start()
main()
yappi.get_func_stats().print_all()
def test_basic(self):
@asyncio.coroutine
def a():
yield from async_sleep(0.1)
burn_cpu(0.2)
@asyncio.coroutine
def b():
yield from a()
@asyncio.coroutine
def recursive_a(n):
if not n:
return
burn_io(0.1)
yield from async_sleep(0.1)
yield from recursive_a(n - 1)
def tag_cbk():
cthread = threading.current_thread()
try:
return cthread._tag
except:
return -1
yappi.set_clock_type("wall")
threading.current_thread()._tag = 0
yappi.set_tag_callback(tag_cbk)
def _thread_event_loop(loop):
asyncio.set_event_loop(loop)
loop.run_forever()
_TCOUNT = 3
_ctag = 1
ts = []
for i in range(_TCOUNT):
_loop = asyncio.new_event_loop()
t = threading.Thread(target=_thread_event_loop, args=(_loop, ))
t._tag = _ctag
t._loop = _loop
t.start()
ts.append(t)
_ctag += 1
@asyncio.coroutine
def stop_loop():
asyncio.get_event_loop().stop()
@asyncio.coroutine
def driver():
futs = []
fut = asyncio.run_coroutine_threadsafe(a(), ts[0]._loop)
futs.append(fut)
fut = asyncio.run_coroutine_threadsafe(recursive_a(5), ts[1]._loop)
futs.append(fut)
fut = asyncio.run_coroutine_threadsafe(b(), ts[2]._loop)
futs.append(fut)
for fut in futs:
fut.result()
# stop asyncio loops in threads
for t in ts:
asyncio.run_coroutine_threadsafe(stop_loop(), t._loop)
yappi.start()
asyncio.get_event_loop().run_until_complete(driver())
yappi.stop()
traces = yappi.get_func_stats()
t1 = '''
tests/test_asyncio.py:140 driver 1 0.000022 1.015813 1.015813
..ts/test_asyncio.py:103 recursive_a 6/1 0.000071 1.014597 0.169099
async_sleep 7 0.000000 0.714345 0.102049
tests/test_asyncio.py:94 a 2 0.000025 0.610155 0.305078
tests/utils.py:135 burn_io 5 0.000018 0.505879 0.101176
tests/utils.py:126 burn_cpu 2 0.380813 0.404432 0.202216
tests/test_asyncio.py:99 b 1 0.000005 0.305355 0.305355
'''
self.assert_traces_almost_equal(t1, traces)
traces = yappi.get_func_stats(filter={'tag': 2})
t1 = '''
..ts/test_asyncio.py:103 recursive_a 6/1 0.000160 1.062948 0.177158
async_sleep 5 0.000000 0.561803 0.112361
tests/utils.py:135 burn_io 5 0.000035 0.500985 0.100197
'''
self.assert_traces_almost_equal(t1, traces)
