def __init__(self):
settings_manager = SettingsManager() # Set up the settings_manager
max_workers = settings_manager.getint('application', 'max-workers') # Get the max workers from settings manager
profiler_on = settings_manager.getint('debugging', 'profiler-on') # Get whether there is a profiler
absolute = settings_manager.getint('save', 'absolute') # Get whether it's an absolute path
save_path = settings_manager.get('save', 'path') # Get whether it's an absolute path
if not absolute:
save_path = PROJECT_PATH + os.path.sep + save_path
executor = ThreadPoolExecutor(max_workers=max_workers, profiler_on=profiler_on) # Set up the thread executor
dis = Disassembler(settings_manager) # Build the disassembler
server = PyDAServer('0.0.0.0',9000) # Set up the PyDA server
save_manager = SaveManager(save_path)
if profiler_on:
profile = Profile()
profile.enable()
app.build_and_run(settings_manager, dis, executor, server, save_manager) # Run the interface
if profiler_on:
profile.disable()
stats = executor.getProfileStats()
if stats == None:
stats = Stats(profile)
else:
stats.add(profile)
with open('profile.stats', 'wb') as statsfile:
stats.stream = statsfile
stats.sort_stats('cumulative').print_stats()
def profile(to=None, sort_by='cumtime'):
'''Profiles a chunk of code, use with the ``with`` statement::
from halonctl.debug import profile
with profile('~/Desktop/stats'):
pass # Do something performance-critical here...
Results for individual runs are collected into ``to``. The specifics of how
reports are done varies depending on what type ``to`` is.
* **File-like objects**: Stats are dumped, according to ``sort_by``, into the stream, separated by newlines - watch out, the file/buffer may grow very big when used in loops.
* **List-like objects**: A number of pstats.Stats objects are appended.
* **str and unicode**: Treated as a path and opened for appending. Tildes (~) will be expanded, and intermediary directories created if possible.
* **None or omitted**: Results are printed to sys.stderr.
'''
if isinstance(to, six.string_types):
to = open_fuzzy(to, 'a')
to_is_stream = hasattr(to, 'write')
to_is_list = hasattr(to, 'append')
p = Profile()
p.enable()
yield
p.disable()
ps = Stats(p, stream=to if to_is_stream else sys.stderr)
ps.sort_stats('cumtime')
if to_is_stream or to is None:
ps.print_stats()
elif to_is_list:
to.append(ps)
def profile(filename, log=None):
"""!Context manager for profiling with cProfile
@param filename filename to which to write profile (profiling disabled if None or empty)
@param log log object for logging the profile operations
If profiling is enabled, the context manager returns the cProfile.Profile object (otherwise
it returns None), which allows additional control over profiling. You can obtain this using
the "as" clause, e.g.:
with profile(filename) as prof:
runYourCodeHere()
The output cumulative profile can be printed with a command-line like:
python -c 'import pstats; pstats.Stats("<filename>").sort_stats("cumtime").print_stats(30)'
"""
if not filename:
# Nothing to do
yield
return
from cProfile import Profile
prof = Profile()
if log is not None:
log.info("Enabling cProfile profiling")
prof.enable()
yield prof
prof.disable()
prof.dump_stats(filename)
if log is not None:
log.info("cProfile stats written to %s" % filename)
def _run_unit(self, unit, set_title=False):
try:
if set_title:
setproctitle('coordinate worker {0!r} {1!r}'
.format(unit.work_spec_name, unit.key))
profiler = None
if ((self.profile_destination and
unit.work_spec_name in self.profile_work_specs)):
now = datetime.now()
unit_info = {
'work_spec_name': unit.work_spec_name,
'work_unit_key': unit.key,
'ymd': now.strftime('%Y%m%d'),
'hms': now.strftime('%H%M%S'),
}
destination = self.profile_destination % unit_info
profiler = Profile()
profiler.enable()
unit.run()
if profiler:
profiler.disable()
profiler.dump_stats(destination)
unit.finish()
except LostLease:
# We don't own the unit any more so don't try to report on it
logger.warn('Lost Lease on %r %r', unit.work_spec_name, unit.key)
except Exception, exc: # pylint: disable=broad-except
unit.fail(exc)
def prepare_game(cursor_lib_callback, cmdline_args=None):
options = get_commandline_options(cmdline_args)
initialize_files_and_folders()
# locale.setlocale(locale.LC_ALL, "")
init_logger_system()
if options.load:
game = load_game(options.load, cursor_lib_callback)
else:
from game import Game
game = Game(options.game, cursor_lib_callback)
if options.profile:
profiler = Profile()
profiler.enable()
def write_profile():
profiler.disable()
profile_util.write_results_log(profiler)
atexit.register(write_profile)
return game
def main():
config = ConfigParser()
try:
config.read(os.path.expanduser("~/.plug/web.conf"))
except Exception as err:
print "Failed to parse config:", str(err)
return 1
port = 8008
if "baseconfig" in config.sections() and "port" in config.options("baseconfig"):
port = int(config.get("baseconfig", "port"))
if "TREEBUG" in os.environ:
last_int = 0
prof = Profile()
while True:
prof.enable()
try:
serve(port=8008)
except KeyboardInterrupt:
if time.time() - last_int < 1:
break
prof.disable()
output = StringIO()
ps = pstats.Stats(prof, stream=output).sort_stats("cumulative")
ps.print_stats()
print "\n".join([line for line in output.getvalue().splitlines()
if not ("/usr/lib" in line or "{" in line)])
output.close()
prof = Profile()
last_int = time.time()
else:
serve(port=port)
def build_problem(main_img, patterns_imgs, Semiring=SemiringArgminPlusElement):
image = MatrixPointer(list(main_img.getdata()), (main_img.size[0], main_img.size[1]))
patterns = dict()
for p in patterns_imgs:
patterns[p] = MatrixPointer(list(patterns_imgs[p].getdata()), (patterns_imgs[p].size[0], patterns_imgs[p].size[1]))
start = Vertex('start')
end = Vertex('end')
fake_edge = Edge(start, end, Semiring.get_zero())
min_width = min(patterns[p].get_size()[0] for p in patterns)
if min_width > image.get_size()[0]:
return Graph([start, end], fake_edge)
profile = Profile()
profile.enable()
vertices, edges = process_img(image, patterns, start)
profile.disable()
Stats(profile).sort_stats('time').print_stats()
vertices['start'] = [start]
vertices['end'] = [end]
if image.get_size()[0] in vertices:
edges = edges.union(Edge(v, end, Semiring.get_unity())
for v in vertices[image.get_size()[0]])
problem = DynamicProgramming(sum(vertices.values(), []), edges)
problem.set_start(start)
problem.set_finish(end)
return problem
class BenchTestResult(unittest.runner.TextTestResult):
"""
A textual test result formatter that can display additional information
such as profile output and benchmarks.
"""
def __init__(self, arguments, stream, descriptions, verbosity):
super(BenchTestResult, self).__init__(stream, descriptions, verbosity)
settings = arguments.get_settings("test_result")
self._sort = settings.get("profile_sort")
self._limit = settings.get("profile_limit")
self._benchmark = verbosity > 2
self._profiler = None
def startTest(self, test):
super(BenchTestResult, self).startTest(test)
if self._benchmark:
self._profiler = Profile()
self._profiler.enable()
def stopTest(self, test):
super(BenchTestResult, self).stopTest(test)
if self._benchmark:
self._profiler.disable()
stats = Stats(self._profiler)
stats.sort_stats(self._sort)
stats.print_stats(self._limit)
def profile(func, max_epochs=15, use_GPU=False, cprofile=True):
"""Run a profiler on the code."""
if cprofile:
p = Profile()
p.enable()
else:
import pycuda
pycuda.driver.start_profiler()
if func == "mnist":
mnist({'use_GPU': use_GPU, 'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'plotting': False, 'batch_size': 7500,
'CG_iter': 10})
elif func == "integrator":
integrator({'shape': [1, 100, 1], 'layers': hf.nl.Logistic(),
'use_GPU': use_GPU, 'debug': False,
'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'CG_iter': 10},
n_inputs=500, sig_len=200, plots=False)
else:
raise ValueError("Unknown profile function")
if cprofile:
p.disable()
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
class Profiler:
profiler_lock = threading.Lock()
"""Global lock used to serialise profiles."""
started = enabled = False
def disable(self):
if self.enabled:
self.p.disable()
self.enabled = False
stats = pstats.Stats(self.p)
if self.pstats is None:
self.pstats = stats
else:
self.pstats.add(stats)
self.count += 1
def enable(self):
if not self.started:
self.start()
elif not self.enabled:
self.p = Profile()
self.p.enable(subcalls=True)
self.enabled = True
def start(self):
"""Start profiling.
"""
if self.started:
return
self.count = 0
self.pstats = None
self.started = True
self.profiler_lock.acquire(True) # Blocks.
try:
self.enable()
except:
self.profiler_lock.release()
self.started = False
raise
def stop(self):
"""Stop profiling.
This unhooks from threading and cleans up the profiler, returning
the gathered Stats object.
:return: A bzrlib.lsprof.Stats object.
"""
try:
self.disable()
p = self.p
del self.p
return Stats(self.pstats, p.getstats(), self.count)
finally:
self.profiler_lock.release()
self.started = False
def profiled(*args, **kargs):
profile = Profile()
profile.enable()
func(*args, **kargs)
profile.disable()
ident = current_thread().ident
profile.dump_stats("/tmp/%s.%s.%i.pstat" % (
hs.hostname, func.__name__, ident
))
def profile(func, file_path):
pr = Profile()
pr.enable()
func()
pr.disable()
s = open(file_path, "w")
sortby = "cumulative"
ps = Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
def wrapped(*args, **kwargs):
if param:
pr = Profile()
pr.enable()
result = func(*args, **kwargs)
pr.disable()
pr.dump_stats(func.__name__ + ".cprofile")
else:
result = func(*args, **kwargs)
return result
def profiled_func(*args, **kwargs):
p = Profile()
try:
# profile the input function
p.enable()
r = f(*args, **kwargs)
p.disable()
return r
finally:
p.print_stats()
def create(
self, commit: Callable[[Mapping[Partition, int]], None]
) -> ProcessingStrategy[TPayload]:
profiler = Profile()
profiler.enable()
return ProcessingStrategyProfilerWrapper(
self.__strategy_factory.create(commit),
profiler,
str(self.__output_directory / f"{int(time.time() * 1000)}.prof"),
)
def apply_some_filters_profile():
frames = tests.input_bulk(200)
profiler = Profile()
profiler.enable()
augmentation.apply_some_filters(frames, tests.config.framerate)
profiler.disable()
profiler.print_stats()
def run_with_profile():
profile = Profile()
profile.enable()
yield profile
profile.disable()
with NamedTemporaryFile() as statsfile:
profile.dump_stats(statsfile.name)
statsfile.flush()
statsfile.seek(0)
profile.stats = base64.b64encode(statsfile.read()).decode()
def test_profile_rdg_generation():
profiler = Profile()
profiler.enable()
for _ in range(5):
level = Level(generation_type=LevelGen.Dungeon)
generate_tiles_to(level)
profiler.disable()
profile_util.write_results_log(profiler)
def main() -> None:
with open("commits copy.txt", "r") as f:
commits = tuple(line.strip() for line in f)
profile = Profile()
profile.enable()
for commit in commits[:50]:
analyze(commit)
profile.disable()
profile_stats = Stats(profile).sort_stats("cumtime")
profile_stats.print_stats()
def profiled_funk(*args, **kwargs):
profile = Profile()
try:
profile.enable()
ret_val = funk(*args, **kwargs)
profile.disable()
finally:
print("__CPROFILE__")
profile.print_stats()
return ret_val
def wrapper(*args, **kwargs):
pr = Profile()
pr.enable()
result = func(*args, **kwargs)
pr.disable()
s = StringIO()
ps = Stats(pr, stream=s).sort_stats('cumulative')
ps.print_stats()
print(s.getvalue())
return result
class CallAspect(MeasureAspect):
def __init__(self, config, name):
super(TimingMeasurement, self).__init__(config, name)
self.typ = "CALL"
self.profile = Profile()
def start(self):
self.profile.enable()
def end(self):
self._data = Stats(self.profile).total_call
return {self.typ: self._data}
def profile(func, max_epochs=1, use_GPU=False, cprofile=True):
"""Run a profiler on the code.
:param str func: the demo function to be profiled (can be 'mnist' or
'integrator')
:param int max_epochs: maximum number of iterations to run
:param bool use_GPU: run optimization on GPU
:param bool cprofile: if True then run the profiling on the CPU, otherwise
use CUDA profiler
"""
if cprofile:
p = Profile()
p.enable()
else:
import pycuda
pycuda.driver.start_profiler()
if func == "mnist":
mnist({
'use_GPU': use_GPU,
'rng': np.random.RandomState(0)
}, {
'max_epochs': max_epochs,
'plotting': False,
'minibatch_size': 7500,
'CG_iter': 10
})
elif func == "integrator":
integrator(
{
'shape': [1, 100, 1],
'layers': hf.nl.Logistic(),
'use_GPU': use_GPU,
'debug': False,
'rng': np.random.RandomState(0)
}, {
'max_epochs': max_epochs,
'CG_iter': 10
},
n_inputs=500,
sig_len=200,
plots=False)
else:
raise ValueError("Unknown profile function")
if cprofile:
p.disable()
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
def profiled_funk(*args, **kwargs):
"""wrapper funk"""
profile = Profile()
try:
profile.enable()
ret_val = funk(*args, **kwargs)
profile.disable()
finally:
print("__CPROFILE__")
profile.print_stats("cumulative")
return ret_val
def main():
from ..logging.base import setup_context as setup_logging_context
from ..logging.base import teardown as teardown_logging
opts = None
pr = None
debug = False
profile = False
try:
setup_logging_context()
from ..utils.pickle import patch_nipype_unpickler
patch_nipype_unpickler()
from .parser import parse_args
opts, should_run = parse_args()
debug = opts.debug
profile = opts.profile
if profile is True:
from cProfile import Profile
pr = Profile()
pr.enable()
run(opts, should_run)
except Exception as e:
logger.exception("Exception: %s", e, exc_info=True)
if debug:
import pdb
pdb.post_mortem()
finally:
if profile and pr is not None:
pr.disable()
if opts is not None:
pr.dump_stats(
Path(opts.workdir) /
f"profile.{format_current_time():s}.prof")
teardown_logging()
# clean up orphan processes
from ..utils.multiprocessing import terminate
terminate()
def inner(*args, **kwargs) -> Any:
"""Inner decorator function."""
profile = Profile()
profile.enable()
ret_val = function(*args, **kwargs)
profile.disable()
string = StringIO()
sortby = 'cumulative'
stats = Stats(profile, stream=string).sort_stats(sortby)
stats.print_stats()
print(string.getvalue())
return ret_val
def wrapper(*args, **kwargs):
profile = Profile()
profile.enable()
try:
func(*args, **kwargs)
finally:
profile.disable()
try:
thread = current_thread()
profile.dump_stats('profile_%s.%s.%s.log' % (getpid(), thread.name, thread.ident))
except:
logger.exception('Failed to dump stats')
def test(algo, size=10000):
mylist = random_list(size)
profile = Profile()
profile.enable()
result = profile.runcall(algo, mylist)
profile.disable()
stringStream = StringIO.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(profile, stream=stringStream).sort_stats(sortby)
ps.print_stats()
print
print "\t\t\t\t{0} sorting {1} ints".format(algo.__name__, size)
print stringStream.getvalue()
def profile():
from cProfile import Profile
from io import StringIO
import pstats
profiler = Profile()
stats_output = StringIO()
try:
profiler.enable()
yield stats_output
finally:
profiler.disable()
profiler_stats = pstats.Stats(profiler, stream=stats_output).sort_stats(pstats.SortKey.CUMULATIVE)
profiler_stats.print_stats()
class Profile:
def __init__(self, name, directory=None):
self.base = ProfileBase()
self.base.enable()
self.path = get_path(PROFILE_DIR,
name,
directory=directory,
ending='.profile')
def close(self):
self.base.disable()
stats = Stats(self.base)
stats.dump_stats(self.path)
def wrapper(*args, **kwargs):
profile = Profile()
profile.enable()
try:
func(*args, **kwargs)
finally:
profile.disable()
try:
thread = current_thread()
profile.dump_stats('profile_%s.%s.%s.log' %
(getpid(), thread.name, thread.ident))
except:
logger.exception('Failed to dump stats')
def wrapper(*args, **kwargs):
prof = Profile()
try:
prof.enable()
execute = method(*args, **kwargs)
prof.disable()
return execute
finally:
stats = StatsDict(prof)
prof_data = stats.sort_stats('cumtime').dump(short)
prof_data['method'] = method.__name__
prof_data['module'] = sys.modules[method.__module__].__file__
manager.add(prof_data)
def wrapper(*args, **kwargs):
"""
wrapper construct
"""
profile = Profile()
profile.enable()
result = func(*args, **kwargs)
profile.disable()
string_io = StringIO()
stats = Stats(profile, stream=string_io).sort_stats(sortby)
stats.print_stats()
LOG.info(string_io.getvalue())
return result
def profiler(enable, outfile):
try:
if enable:
profiler = Profile()
profiler.enable()
yield
finally:
if enable:
profiler.disable()
stats = Stats(profiler)
stats.sort_stats('tottime')
stats.dump_stats(outfile)
def wrapper(*args, **kwargs):
pr = Profile()
pr.enable()
result = func(*args, **kwargs)
pr.disable()
s = StringIO()
ps = Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
log.info(s.getvalue())
return result
def run(self):
profile = Profile()
profile.enable()
if self.isCanceled():
return
successful = False
self.current_sync = timing.unix_timestamp()
# Delete playlist and video node files from Kodi
utils.delete_playlists()
utils.delete_nodes()
# Get latest Plex libraries and build playlist and video node files
if not sections.sync_from_pms():
return
try:
# Fire up our single processing thread
self.queue = backgroundthread.Queue.Queue(maxsize=1000)
self.processing_thread = ProcessMetadata(self.queue,
self.current_sync,
self.show_dialog)
self.processing_thread.start()
# Actual syncing - do only new items first
LOG.info('Running full_library_sync with repair=%s',
self.repair)
if not self.full_library_sync():
return
# Tell the processing thread to exit with one last element None
self.queue.put(None)
if self.isCanceled():
return
if PLAYLIST_SYNC_ENABLED and not playlists.full_sync():
return
successful = True
except:
utils.ERROR(txt='full_sync.py crashed', notify=True)
finally:
# This will block until the processing thread really exits
LOG.debug('Waiting for processing thread to exit')
self.processing_thread.join()
common.update_kodi_library(video=True, music=True)
self.threader.shutdown()
if self.callback:
self.callback(successful)
LOG.info('Done full_sync')
profile.disable()
string_io = StringIO()
stats = Stats(profile, stream=string_io).sort_stats('cumulative')
stats.print_stats()
LOG.info('cProfile result: ')
LOG.info(string_io.getvalue())
def enable_time_profiler():
"""
Enable time profiler.
:return: cProfile.Profile
"""
global parsed_args
pr = None
if parsed_args.profile:
pr = Profile()
pr.enable()
return pr
def enable_time_profiler():
"""
Enable time profiler.
:return: cProfile.Profile
"""
global parsed_args
pr = None
if parsed_args.profile:
pr = Profile()
pr.enable()
return pr
def startup(dir):
dir = abspath(clean_path(dir))
log.info(f'Will save profile data to {dir} dir')
if not exists(dir):
makedirs(dir, exist_ok=True)
if not isdir(dir):
raise Exception(f'{dir} is not a path')
if glob(f'{dir}/*'):
log.warning(f'{dir} already contains data')
makedirs(join(dir, START), exist_ok=True)
makedirs(join(dir, DURATION), exist_ok=True)
profiler = Profile()
profiler.enable()
return profiler, dir
def runAndTimeSRM(srm, srm_type, make_plot=False):
print("Testing {} SRM model!".format(srm_type))
prof = Profile()
prof.enable()
P, T, A, x = burnSRM(srm, srm_type, make_plot)
prof.disable()
t = delta_t * np.arange(P.size)
prof.dump_stats('{}_srm.stats'.format(srm_type))
return P, T, A, x, t
def decorated(*pargs, **kwargs):
try:
pr = Profile()
pr.enable()
res = f(*pargs, **kwargs)
pr.disable()
return res
except Exception:
raise
finally:
# Print results
s = StringIO()
ps = Stats(pr, stream=s).sort_stats('cumulative')
ps.print_stats()
print(s.getvalue())
def pytest_runtest_call(item):
if SHOULD_PROFILE:
p = Profile()
p.enable()
yield
p.disable()
stats = Stats(p)
if SHOULD_PRINT:
stats.sort_stats('cumulative').print_stats(50)
if SHOULD_STORE:
if not os.path.exists(BASEDIR):
os.mkdir(BASEDIR)
p.dump_stats(os.path.join(BASEDIR, '%s.pkl' % item.name))
else:
yield
def _wrap(*kl, **kw):
prof = Profile()
prof.enable()
try:
return func(*kl, **kw)
finally:
prof.disable()
result = StringIO()
ps = Stats(prof, stream = result).sort_stats('cumulative')
ps.print_stats()
print result.getvalue()
def prof_context(sort_by="cumtime", limit=10, timer=time.perf_counter):
"""
参考了雨痕的 <Python学习笔记>
:param sort_by:
:param limit:
:param timer:
:return:
"""
p = Profile()
p.enable()
try:
yield
finally:
p.disable()
s = pstats.Stats(p).sort_stats(sort_by)
s.print_stats(limit)
def prof_main():
from cProfile import Profile
prof = Profile()
prof.enable()
fish = factory.run_silent_game(player_count=6)
prof.disable()
line_sep_char = '#'
win_string = '{1} Winners: {0}'.format(
', '.join(['Player ' + p.name for p in fish.winner]), line_sep_char)
sep = line_sep_char * len(win_string)
print('\n{0}\n{1}\n{0}'.format(sep, win_string))
prof.print_stats()
return fish
def do(impl, count):
# warm-up
_res = [impl() for _ in _range(count)]
profile = Profile()
profile.enable()
res = [impl() for _ in _range(count)]
profile.disable()
out = StringIO()
stats = Stats(profile, stream=out)
stats.strip_dirs()
stats.sort_stats('calls').print_stats(10)
print(out.getvalue().lstrip())
out.close()
return _res, res
def do(impl, count):
# warm-up
_res = [impl() for _ in _range(count)]
profile = Profile()
profile.enable()
res = [impl() for _ in _range(count)]
profile.disable()
out = StringIO()
stats = Stats(profile, stream=out)
stats.strip_dirs()
stats.sort_stats('calls').print_stats(10)
print(out.getvalue().lstrip())
out.close()
return _res, res
def profile(func, max_epochs=15, use_GPU=False, cprofile=True):
"""Run a profiler on the code."""
if cprofile:
p = Profile()
p.enable()
else:
import pycuda
pycuda.driver.start_profiler()
if func == "mnist":
mnist({
'use_GPU': use_GPU,
'rng': np.random.RandomState(0)
}, {
'max_epochs': max_epochs,
'plotting': False,
'batch_size': 7500,
'CG_iter': 10
})
elif func == "integrator":
integrator(
{
'shape': [1, 100, 1],
'layers': hf.nl.Logistic(),
'use_GPU': use_GPU,
'debug': False,
'rng': np.random.RandomState(0)
}, {
'max_epochs': max_epochs,
'CG_iter': 10
},
n_inputs=500,
sig_len=200,
plots=False)
else:
raise ValueError("Unknown profile function")
if cprofile:
p.disable()
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
def prof_main():
from cProfile import Profile
prof = Profile()
prof.enable()
fish = BasicGoFish(build_players())
fish.do_full_round()
prof.disable()
line_sep_char = '#'
win_string = '{1} Winners: {0}'.format(
', '.join(['Player ' + p.name for p in fish.winner]), line_sep_char)
sep = line_sep_char * len(win_string)
print('\n{0}\n{1}\n{0}'.format(sep, win_string))
prof.print_stats()
return fish
class BenchmarkThread(Thread):
def __init__(self, thread_num, session, query, values, num_queries, profile):
Thread.__init__(self)
self.thread_num = thread_num
self.session = session
self.query = query
self.values = values
self.num_queries = num_queries
self.profiler = Profile() if profile else None
def start_profile(self):
if self.profiler:
self.profiler.enable()
def finish_profile(self):
if self.profiler:
self.profiler.disable()
self.profiler.dump_stats("profile-%d" % self.thread_num)
def some_func(foo, bar):
"""
:param foo:
:param bar:
:return:
"""
try:
profile1 = Profile()
profile1.enable() # to measure only a part of a function
result = foo + bar
profile1.disable()
stats1 = pstats.Stats(profile)
stats.print_stats()
return result
except Exception as exception:
raise exception
def __init_profiler(
profiler_arg: bool,
logger: logging.Logger) -> Tuple[Optional[Profile], Optional[str]]:
"""
Initialise profiling environment by creating a cprofile.Profiler instance, a folder where pstats can be dumped
Args:
profiler_arg: the value of profiler argument passed when running the command
logger: a logger instance
Returns:
If profiling enabled, a tuple of profiler instance and profiling directory where the stats files
would be dumped, otherwise, a tuple of nulls
"""
if profiler_arg:
logger.info('Profiling mode enabled')
logger.debug('Creating & enabling profiler ...')
profiler = Profile()
profiler.enable()
logger.debug('Profiler created.')
profiling_dir = os.path.join(
PROFILING_DIR,
f'{datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f")}_{generate_random_string(10)}',
)
try:
os.makedirs(profiling_dir)
logger.debug('Profiling directory "%s" created', profiling_dir)
except OSError as ex:
if ex.errno != errno.EEXIST:
raise
logger.debug('Profiling directory "%s" already exists',
profiling_dir)
return profiler, profiling_dir
logger.info('Profiling mode not enabled')
return None, None
def compare_phi():
"""
Problem 38
>>> a, b = compare_phi()
>>> a > b
True
"""
from cProfile import Profile
from pstats import Stats
pr_normal = Profile()
pr_normal.enable()
phi(10090)
pr_normal.disable()
time_normal = Stats(pr_normal).total_tt
pr_improved = Profile()
pr_improved.enable()
phi_improved(10090)
pr_improved.disable()
time_improved = Stats(pr_improved).total_tt
return time_normal, time_improved
def profile_rnn_calc_G(cprofile=True):
"""Run a profiler on the recurrent curvature calculation.
:param bool cprofile: use True if profiling on the CPU, False if using the
CUDA profiler
"""
inputs = np.random.randn(1024, 128, 1).astype(np.float32)
targets = np.random.randn(1024, 128, 1).astype(np.float32)
N = 128
rnn = hf.RNNet([1, N, 1], use_GPU=True)
rnn.optimizer = hf.opt.HessianFree() # for struc_damping check
rnn.cache_minibatch(inputs, targets)
v = np.random.randn(rnn.W.size).astype(np.float32)
for _ in range(2):
# run it a few times to get rid of any startup overhead
rnn.GPU_calc_G(v)
if cprofile:
start = time.time()
p = Profile()
p.enable()
else:
pycuda.driver.start_profiler()
for _ in range(100):
_ = rnn.GPU_calc_G(v)
if cprofile:
p.disable()
print("time", time.time() - start)
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
def test_ensemble(self,test_file):
assert self.authenticated, 'Not authenticated!'
# download a local copy of the ensemble
self.logger.info('Creating local ensemble')
local_ensemble = Ensemble(self.ensemble_res,api=self.api)
# make the Fields object
source = self.api.get_source(self.source_res)
fields = Fields(source['object']['fields'])
self.logger.info('Reading test data and generating predictions')
true_labels = []
predict_labels = []
pr = Profile()
pr.enable()
with open(test_file) as fid:
test_reader = csv.reader(fid)
# skip the header line
test_reader.next()
for row in test_reader:
row_list = [val for val in row]
true_labels.append(row_list.pop())
instance = fields.pair(row_list)
predict_labels.append(local_ensemble.predict(instance,
by_name=False,
method=1))
pr.disable()
ps = Stats(pr)
self.predict_time = ps.total_tt
# eval_args = {'combiner':1}
# evaluation = self.api.create_evaluation(self.ensemble_res,test_data,eval_args)
# check_resource(evaluation['resource'],self.api.get_evaluation)
# evaluation = self.api.get_evaluation(evaluation['resource'])
# matrix = evaluation['object']['result']['model']['confusion_matrix']
# self.predict_time = evaluation['object']['status']['elapsed']/1000
if self.regression:
self.results = (predict_labels,true_labels)
else:
self.results = make_confusion_matrix(true_labels,predict_labels)
def profile_calc_G(cprofile=True):
"""Run a profiler on the feedforward curvature calculation.
:param bool cprofile: use True if profiling on the CPU, False if using the
CUDA profiler
"""
inputs = np.random.randn(1024, 1).astype(np.float32)
targets = np.random.randn(1024, 1).astype(np.float32)
N = 1024
ff = hf.FFNet([1, N, N, 1], use_GPU=True)
ff.cache_minibatch(inputs, targets)
v = np.random.randn(ff.W.size).astype(np.float32)
for _ in range(5):
# run it a few times to get rid of any startup overhead
ff.GPU_calc_G(v)
if cprofile:
start = time.time()
p = Profile()
p.enable()
else:
pycuda.driver.start_profiler()
for _ in range(500):
_ = ff.GPU_calc_G(v)
if cprofile:
p.disable()
print("time", time.time() - start)
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
def profile_dot(cprofile=True):
"""Run a profiler on the matrix multiplication kernel.
:param bool cprofile: use True if profiling on the CPU, False if using the
CUDA profiler
"""
N = 1024
a = np.random.randn(N, N).astype(np.float32)
b = np.random.randn(N, N).astype(np.float32)
a_gpu = gpuarray.to_gpu(a)
b_gpu = gpuarray.to_gpu(b)
c_gpu = gpuarray.zeros((N, N), np.float32)
for _ in range(2):
# run it a few times to get rid of any startup overhead
hf.gpu.dot(a_gpu, b_gpu, out=c_gpu)
if cprofile:
start = time.time()
p = Profile()
p.enable()
else:
pycuda.autoinit.context.synchronize()
pycuda.driver.start_profiler()
for _ in range(100):
hf.gpu.dot(a_gpu, b_gpu, out=c_gpu, transpose_a=True,
transpose_b=True)
c_gpu.get()
if cprofile:
p.disable()
print("time", time.time() - start)
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
class BenchmarkThread(Thread):
def __init__(self, thread_num, session, query, values, num_queries, protocol_version, profile):
Thread.__init__(self)
self.thread_num = thread_num
self.session = session
self.query = query
self.values = values
self.num_queries = num_queries
self.protocol_version = protocol_version
self.profiler = Profile() if profile else None
def start_profile(self):
if self.profiler:
self.profiler.enable()
def run_query(self, key, **kwargs):
return self.session.execute_async(self.query.format(key=key), **kwargs)
def finish_profile(self):
if self.profiler:
self.profiler.disable()
self.profiler.dump_stats("profile-%d" % self.thread_num)
def profile(func, max_epochs=1, use_GPU=False, cprofile=True):
"""Run a profiler on the code.
:param str func: the demo function to be profiled (can be 'mnist' or
'integrator')
:param int max_epochs: maximum number of iterations to run
:param bool use_GPU: run optimization on GPU
:param bool cprofile: if True then run the profiling on the CPU, otherwise
use CUDA profiler
"""
if cprofile:
p = Profile()
p.enable()
else:
import pycuda
pycuda.driver.start_profiler()
if func == "mnist":
mnist({'use_GPU': use_GPU, 'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'plotting': False,
'minibatch_size': 7500, 'CG_iter': 10})
elif func == "integrator":
integrator({'shape': [1, 100, 1], 'layers': hf.nl.Logistic(),
'use_GPU': use_GPU, 'debug': False,
'rng': np.random.RandomState(0)},
{'max_epochs': max_epochs, 'CG_iter': 10},
n_inputs=500, sig_len=200, plots=False)
else:
raise ValueError("Unknown profile function")
if cprofile:
p.disable()
ps = pstats.Stats(p)
ps.strip_dirs().sort_stats('time').print_stats(20)
else:
pycuda.driver.stop_profiler()
