def __init__(self): settings_manager = SettingsManager() # Set up the settings_manager max_workers = settings_manager.getint('application', 'max-workers') # Get the max workers from settings manager profiler_on = settings_manager.getint('debugging', 'profiler-on') # Get whether there is a profiler absolute = settings_manager.getint('save', 'absolute') # Get whether it's an absolute path save_path = settings_manager.get('save', 'path') # Get whether it's an absolute path if not absolute: save_path = PROJECT_PATH + os.path.sep + save_path executor = ThreadPoolExecutor(max_workers=max_workers, profiler_on=profiler_on) # Set up the thread executor dis = Disassembler(settings_manager) # Build the disassembler server = PyDAServer('0.0.0.0',9000) # Set up the PyDA server save_manager = SaveManager(save_path) if profiler_on: profile = Profile() profile.enable() app.build_and_run(settings_manager, dis, executor, server, save_manager) # Run the interface if profiler_on: profile.disable() stats = executor.getProfileStats() if stats == None: stats = Stats(profile) else: stats.add(profile) with open('profile.stats', 'wb') as statsfile: stats.stream = statsfile stats.sort_stats('cumulative').print_stats()
def profile(to=None, sort_by='cumtime'): '''Profiles a chunk of code, use with the ``with`` statement:: from halonctl.debug import profile with profile('~/Desktop/stats'): pass # Do something performance-critical here... Results for individual runs are collected into ``to``. The specifics of how reports are done varies depending on what type ``to`` is. * **File-like objects**: Stats are dumped, according to ``sort_by``, into the stream, separated by newlines - watch out, the file/buffer may grow very big when used in loops. * **List-like objects**: A number of pstats.Stats objects are appended. * **str and unicode**: Treated as a path and opened for appending. Tildes (~) will be expanded, and intermediary directories created if possible. * **None or omitted**: Results are printed to sys.stderr. ''' if isinstance(to, six.string_types): to = open_fuzzy(to, 'a') to_is_stream = hasattr(to, 'write') to_is_list = hasattr(to, 'append') p = Profile() p.enable() yield p.disable() ps = Stats(p, stream=to if to_is_stream else sys.stderr) ps.sort_stats('cumtime') if to_is_stream or to is None: ps.print_stats() elif to_is_list: to.append(ps)
def profile(filename, log=None): """!Context manager for profiling with cProfile @param filename filename to which to write profile (profiling disabled if None or empty) @param log log object for logging the profile operations If profiling is enabled, the context manager returns the cProfile.Profile object (otherwise it returns None), which allows additional control over profiling. You can obtain this using the "as" clause, e.g.: with profile(filename) as prof: runYourCodeHere() The output cumulative profile can be printed with a command-line like: python -c 'import pstats; pstats.Stats("<filename>").sort_stats("cumtime").print_stats(30)' """ if not filename: # Nothing to do yield return from cProfile import Profile prof = Profile() if log is not None: log.info("Enabling cProfile profiling") prof.enable() yield prof prof.disable() prof.dump_stats(filename) if log is not None: log.info("cProfile stats written to %s" % filename)
def _run_unit(self, unit, set_title=False): try: if set_title: setproctitle('coordinate worker {0!r} {1!r}' .format(unit.work_spec_name, unit.key)) profiler = None if ((self.profile_destination and unit.work_spec_name in self.profile_work_specs)): now = datetime.now() unit_info = { 'work_spec_name': unit.work_spec_name, 'work_unit_key': unit.key, 'ymd': now.strftime('%Y%m%d'), 'hms': now.strftime('%H%M%S'), } destination = self.profile_destination % unit_info profiler = Profile() profiler.enable() unit.run() if profiler: profiler.disable() profiler.dump_stats(destination) unit.finish() except LostLease: # We don't own the unit any more so don't try to report on it logger.warn('Lost Lease on %r %r', unit.work_spec_name, unit.key) except Exception, exc: # pylint: disable=broad-except unit.fail(exc)
def prepare_game(cursor_lib_callback, cmdline_args=None): options = get_commandline_options(cmdline_args) initialize_files_and_folders() # locale.setlocale(locale.LC_ALL, "") init_logger_system() if options.load: game = load_game(options.load, cursor_lib_callback) else: from game import Game game = Game(options.game, cursor_lib_callback) if options.profile: profiler = Profile() profiler.enable() def write_profile(): profiler.disable() profile_util.write_results_log(profiler) atexit.register(write_profile) return game
def main(): config = ConfigParser() try: config.read(os.path.expanduser("~/.plug/web.conf")) except Exception as err: print "Failed to parse config:", str(err) return 1 port = 8008 if "baseconfig" in config.sections() and "port" in config.options("baseconfig"): port = int(config.get("baseconfig", "port")) if "TREEBUG" in os.environ: last_int = 0 prof = Profile() while True: prof.enable() try: serve(port=8008) except KeyboardInterrupt: if time.time() - last_int < 1: break prof.disable() output = StringIO() ps = pstats.Stats(prof, stream=output).sort_stats("cumulative") ps.print_stats() print "\n".join([line for line in output.getvalue().splitlines() if not ("/usr/lib" in line or "{" in line)]) output.close() prof = Profile() last_int = time.time() else: serve(port=port)
def build_problem(main_img, patterns_imgs, Semiring=SemiringArgminPlusElement): image = MatrixPointer(list(main_img.getdata()), (main_img.size[0], main_img.size[1])) patterns = dict() for p in patterns_imgs: patterns[p] = MatrixPointer(list(patterns_imgs[p].getdata()), (patterns_imgs[p].size[0], patterns_imgs[p].size[1])) start = Vertex('start') end = Vertex('end') fake_edge = Edge(start, end, Semiring.get_zero()) min_width = min(patterns[p].get_size()[0] for p in patterns) if min_width > image.get_size()[0]: return Graph([start, end], fake_edge) profile = Profile() profile.enable() vertices, edges = process_img(image, patterns, start) profile.disable() Stats(profile).sort_stats('time').print_stats() vertices['start'] = [start] vertices['end'] = [end] if image.get_size()[0] in vertices: edges = edges.union(Edge(v, end, Semiring.get_unity()) for v in vertices[image.get_size()[0]]) problem = DynamicProgramming(sum(vertices.values(), []), edges) problem.set_start(start) problem.set_finish(end) return problem
class BenchTestResult(unittest.runner.TextTestResult): """ A textual test result formatter that can display additional information such as profile output and benchmarks. """ def __init__(self, arguments, stream, descriptions, verbosity): super(BenchTestResult, self).__init__(stream, descriptions, verbosity) settings = arguments.get_settings("test_result") self._sort = settings.get("profile_sort") self._limit = settings.get("profile_limit") self._benchmark = verbosity > 2 self._profiler = None def startTest(self, test): super(BenchTestResult, self).startTest(test) if self._benchmark: self._profiler = Profile() self._profiler.enable() def stopTest(self, test): super(BenchTestResult, self).stopTest(test) if self._benchmark: self._profiler.disable() stats = Stats(self._profiler) stats.sort_stats(self._sort) stats.print_stats(self._limit)
def profile(func, max_epochs=15, use_GPU=False, cprofile=True): """Run a profiler on the code.""" if cprofile: p = Profile() p.enable() else: import pycuda pycuda.driver.start_profiler() if func == "mnist": mnist({'use_GPU': use_GPU, 'rng': np.random.RandomState(0)}, {'max_epochs': max_epochs, 'plotting': False, 'batch_size': 7500, 'CG_iter': 10}) elif func == "integrator": integrator({'shape': [1, 100, 1], 'layers': hf.nl.Logistic(), 'use_GPU': use_GPU, 'debug': False, 'rng': np.random.RandomState(0)}, {'max_epochs': max_epochs, 'CG_iter': 10}, n_inputs=500, sig_len=200, plots=False) else: raise ValueError("Unknown profile function") if cprofile: p.disable() ps = pstats.Stats(p) ps.strip_dirs().sort_stats('time').print_stats(20) else: pycuda.driver.stop_profiler()
class Profiler: profiler_lock = threading.Lock() """Global lock used to serialise profiles.""" started = enabled = False def disable(self): if self.enabled: self.p.disable() self.enabled = False stats = pstats.Stats(self.p) if self.pstats is None: self.pstats = stats else: self.pstats.add(stats) self.count += 1 def enable(self): if not self.started: self.start() elif not self.enabled: self.p = Profile() self.p.enable(subcalls=True) self.enabled = True def start(self): """Start profiling. """ if self.started: return self.count = 0 self.pstats = None self.started = True self.profiler_lock.acquire(True) # Blocks. try: self.enable() except: self.profiler_lock.release() self.started = False raise def stop(self): """Stop profiling. This unhooks from threading and cleans up the profiler, returning the gathered Stats object. :return: A bzrlib.lsprof.Stats object. """ try: self.disable() p = self.p del self.p return Stats(self.pstats, p.getstats(), self.count) finally: self.profiler_lock.release() self.started = False
def profiled(*args, **kargs): profile = Profile() profile.enable() func(*args, **kargs) profile.disable() ident = current_thread().ident profile.dump_stats("/tmp/%s.%s.%i.pstat" % ( hs.hostname, func.__name__, ident ))
def profile(func, file_path): pr = Profile() pr.enable() func() pr.disable() s = open(file_path, "w") sortby = "cumulative" ps = Stats(pr, stream=s).sort_stats(sortby) ps.print_stats()
def wrapped(*args, **kwargs): if param: pr = Profile() pr.enable() result = func(*args, **kwargs) pr.disable() pr.dump_stats(func.__name__ + ".cprofile") else: result = func(*args, **kwargs) return result
def profiled_func(*args, **kwargs): p = Profile() try: # profile the input function p.enable() r = f(*args, **kwargs) p.disable() return r finally: p.print_stats()
def create( self, commit: Callable[[Mapping[Partition, int]], None] ) -> ProcessingStrategy[TPayload]: profiler = Profile() profiler.enable() return ProcessingStrategyProfilerWrapper( self.__strategy_factory.create(commit), profiler, str(self.__output_directory / f"{int(time.time() * 1000)}.prof"), )
def apply_some_filters_profile(): frames = tests.input_bulk(200) profiler = Profile() profiler.enable() augmentation.apply_some_filters(frames, tests.config.framerate) profiler.disable() profiler.print_stats()
def run_with_profile(): profile = Profile() profile.enable() yield profile profile.disable() with NamedTemporaryFile() as statsfile: profile.dump_stats(statsfile.name) statsfile.flush() statsfile.seek(0) profile.stats = base64.b64encode(statsfile.read()).decode()
def test_profile_rdg_generation(): profiler = Profile() profiler.enable() for _ in range(5): level = Level(generation_type=LevelGen.Dungeon) generate_tiles_to(level) profiler.disable() profile_util.write_results_log(profiler)
def main() -> None: with open("commits copy.txt", "r") as f: commits = tuple(line.strip() for line in f) profile = Profile() profile.enable() for commit in commits[:50]: analyze(commit) profile.disable() profile_stats = Stats(profile).sort_stats("cumtime") profile_stats.print_stats()
def profiled_funk(*args, **kwargs): profile = Profile() try: profile.enable() ret_val = funk(*args, **kwargs) profile.disable() finally: print("__CPROFILE__") profile.print_stats() return ret_val
def wrapper(*args, **kwargs): pr = Profile() pr.enable() result = func(*args, **kwargs) pr.disable() s = StringIO() ps = Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() print(s.getvalue()) return result
class CallAspect(MeasureAspect): def __init__(self, config, name): super(TimingMeasurement, self).__init__(config, name) self.typ = "CALL" self.profile = Profile() def start(self): self.profile.enable() def end(self): self._data = Stats(self.profile).total_call return {self.typ: self._data}
def profile(func, max_epochs=1, use_GPU=False, cprofile=True): """Run a profiler on the code. :param str func: the demo function to be profiled (can be 'mnist' or 'integrator') :param int max_epochs: maximum number of iterations to run :param bool use_GPU: run optimization on GPU :param bool cprofile: if True then run the profiling on the CPU, otherwise use CUDA profiler """ if cprofile: p = Profile() p.enable() else: import pycuda pycuda.driver.start_profiler() if func == "mnist": mnist({ 'use_GPU': use_GPU, 'rng': np.random.RandomState(0) }, { 'max_epochs': max_epochs, 'plotting': False, 'minibatch_size': 7500, 'CG_iter': 10 }) elif func == "integrator": integrator( { 'shape': [1, 100, 1], 'layers': hf.nl.Logistic(), 'use_GPU': use_GPU, 'debug': False, 'rng': np.random.RandomState(0) }, { 'max_epochs': max_epochs, 'CG_iter': 10 }, n_inputs=500, sig_len=200, plots=False) else: raise ValueError("Unknown profile function") if cprofile: p.disable() ps = pstats.Stats(p) ps.strip_dirs().sort_stats('time').print_stats(20) else: pycuda.driver.stop_profiler()
def profiled_funk(*args, **kwargs): """wrapper funk""" profile = Profile() try: profile.enable() ret_val = funk(*args, **kwargs) profile.disable() finally: print("__CPROFILE__") profile.print_stats("cumulative") return ret_val
def main(): from ..logging.base import setup_context as setup_logging_context from ..logging.base import teardown as teardown_logging opts = None pr = None debug = False profile = False try: setup_logging_context() from ..utils.pickle import patch_nipype_unpickler patch_nipype_unpickler() from .parser import parse_args opts, should_run = parse_args() debug = opts.debug profile = opts.profile if profile is True: from cProfile import Profile pr = Profile() pr.enable() run(opts, should_run) except Exception as e: logger.exception("Exception: %s", e, exc_info=True) if debug: import pdb pdb.post_mortem() finally: if profile and pr is not None: pr.disable() if opts is not None: pr.dump_stats( Path(opts.workdir) / f"profile.{format_current_time():s}.prof") teardown_logging() # clean up orphan processes from ..utils.multiprocessing import terminate terminate()
def inner(*args, **kwargs) -> Any: """Inner decorator function.""" profile = Profile() profile.enable() ret_val = function(*args, **kwargs) profile.disable() string = StringIO() sortby = 'cumulative' stats = Stats(profile, stream=string).sort_stats(sortby) stats.print_stats() print(string.getvalue()) return ret_val
def wrapper(*args, **kwargs): profile = Profile() profile.enable() try: func(*args, **kwargs) finally: profile.disable() try: thread = current_thread() profile.dump_stats('profile_%s.%s.%s.log' % (getpid(), thread.name, thread.ident)) except: logger.exception('Failed to dump stats')
def test(algo, size=10000): mylist = random_list(size) profile = Profile() profile.enable() result = profile.runcall(algo, mylist) profile.disable() stringStream = StringIO.StringIO() sortby = 'cumulative' ps = pstats.Stats(profile, stream=stringStream).sort_stats(sortby) ps.print_stats() print print "\t\t\t\t{0} sorting {1} ints".format(algo.__name__, size) print stringStream.getvalue()
def profile(): from cProfile import Profile from io import StringIO import pstats profiler = Profile() stats_output = StringIO() try: profiler.enable() yield stats_output finally: profiler.disable() profiler_stats = pstats.Stats(profiler, stream=stats_output).sort_stats(pstats.SortKey.CUMULATIVE) profiler_stats.print_stats()
class Profile: def __init__(self, name, directory=None): self.base = ProfileBase() self.base.enable() self.path = get_path(PROFILE_DIR, name, directory=directory, ending='.profile') def close(self): self.base.disable() stats = Stats(self.base) stats.dump_stats(self.path)
def wrapper(*args, **kwargs): prof = Profile() try: prof.enable() execute = method(*args, **kwargs) prof.disable() return execute finally: stats = StatsDict(prof) prof_data = stats.sort_stats('cumtime').dump(short) prof_data['method'] = method.__name__ prof_data['module'] = sys.modules[method.__module__].__file__ manager.add(prof_data)
def wrapper(*args, **kwargs): """ wrapper construct """ profile = Profile() profile.enable() result = func(*args, **kwargs) profile.disable() string_io = StringIO() stats = Stats(profile, stream=string_io).sort_stats(sortby) stats.print_stats() LOG.info(string_io.getvalue()) return result
def profiler(enable, outfile): try: if enable: profiler = Profile() profiler.enable() yield finally: if enable: profiler.disable() stats = Stats(profiler) stats.sort_stats('tottime') stats.dump_stats(outfile)
def wrapper(*args, **kwargs): pr = Profile() pr.enable() result = func(*args, **kwargs) pr.disable() s = StringIO() ps = Stats(pr, stream=s).sort_stats(sortby) ps.print_stats() log.info(s.getvalue()) return result
def run(self): profile = Profile() profile.enable() if self.isCanceled(): return successful = False self.current_sync = timing.unix_timestamp() # Delete playlist and video node files from Kodi utils.delete_playlists() utils.delete_nodes() # Get latest Plex libraries and build playlist and video node files if not sections.sync_from_pms(): return try: # Fire up our single processing thread self.queue = backgroundthread.Queue.Queue(maxsize=1000) self.processing_thread = ProcessMetadata(self.queue, self.current_sync, self.show_dialog) self.processing_thread.start() # Actual syncing - do only new items first LOG.info('Running full_library_sync with repair=%s', self.repair) if not self.full_library_sync(): return # Tell the processing thread to exit with one last element None self.queue.put(None) if self.isCanceled(): return if PLAYLIST_SYNC_ENABLED and not playlists.full_sync(): return successful = True except: utils.ERROR(txt='full_sync.py crashed', notify=True) finally: # This will block until the processing thread really exits LOG.debug('Waiting for processing thread to exit') self.processing_thread.join() common.update_kodi_library(video=True, music=True) self.threader.shutdown() if self.callback: self.callback(successful) LOG.info('Done full_sync') profile.disable() string_io = StringIO() stats = Stats(profile, stream=string_io).sort_stats('cumulative') stats.print_stats() LOG.info('cProfile result: ') LOG.info(string_io.getvalue())
def enable_time_profiler(): """ Enable time profiler. :return: cProfile.Profile """ global parsed_args pr = None if parsed_args.profile: pr = Profile() pr.enable() return pr
def startup(dir): dir = abspath(clean_path(dir)) log.info(f'Will save profile data to {dir} dir') if not exists(dir): makedirs(dir, exist_ok=True) if not isdir(dir): raise Exception(f'{dir} is not a path') if glob(f'{dir}/*'): log.warning(f'{dir} already contains data') makedirs(join(dir, START), exist_ok=True) makedirs(join(dir, DURATION), exist_ok=True) profiler = Profile() profiler.enable() return profiler, dir
def runAndTimeSRM(srm, srm_type, make_plot=False): print("Testing {} SRM model!".format(srm_type)) prof = Profile() prof.enable() P, T, A, x = burnSRM(srm, srm_type, make_plot) prof.disable() t = delta_t * np.arange(P.size) prof.dump_stats('{}_srm.stats'.format(srm_type)) return P, T, A, x, t
def decorated(*pargs, **kwargs): try: pr = Profile() pr.enable() res = f(*pargs, **kwargs) pr.disable() return res except Exception: raise finally: # Print results s = StringIO() ps = Stats(pr, stream=s).sort_stats('cumulative') ps.print_stats() print(s.getvalue())
def pytest_runtest_call(item): if SHOULD_PROFILE: p = Profile() p.enable() yield p.disable() stats = Stats(p) if SHOULD_PRINT: stats.sort_stats('cumulative').print_stats(50) if SHOULD_STORE: if not os.path.exists(BASEDIR): os.mkdir(BASEDIR) p.dump_stats(os.path.join(BASEDIR, '%s.pkl' % item.name)) else: yield
def _wrap(*kl, **kw): prof = Profile() prof.enable() try: return func(*kl, **kw) finally: prof.disable() result = StringIO() ps = Stats(prof, stream = result).sort_stats('cumulative') ps.print_stats() print result.getvalue()
def prof_context(sort_by="cumtime", limit=10, timer=time.perf_counter): """ 参考了雨痕的 <Python学习笔记> :param sort_by: :param limit: :param timer: :return: """ p = Profile() p.enable() try: yield finally: p.disable() s = pstats.Stats(p).sort_stats(sort_by) s.print_stats(limit)
def prof_main(): from cProfile import Profile prof = Profile() prof.enable() fish = factory.run_silent_game(player_count=6) prof.disable() line_sep_char = '#' win_string = '{1} Winners: {0}'.format( ', '.join(['Player ' + p.name for p in fish.winner]), line_sep_char) sep = line_sep_char * len(win_string) print('\n{0}\n{1}\n{0}'.format(sep, win_string)) prof.print_stats() return fish
def do(impl, count): # warm-up _res = [impl() for _ in _range(count)] profile = Profile() profile.enable() res = [impl() for _ in _range(count)] profile.disable() out = StringIO() stats = Stats(profile, stream=out) stats.strip_dirs() stats.sort_stats('calls').print_stats(10) print(out.getvalue().lstrip()) out.close() return _res, res
def profile(func, max_epochs=15, use_GPU=False, cprofile=True): """Run a profiler on the code.""" if cprofile: p = Profile() p.enable() else: import pycuda pycuda.driver.start_profiler() if func == "mnist": mnist({ 'use_GPU': use_GPU, 'rng': np.random.RandomState(0) }, { 'max_epochs': max_epochs, 'plotting': False, 'batch_size': 7500, 'CG_iter': 10 }) elif func == "integrator": integrator( { 'shape': [1, 100, 1], 'layers': hf.nl.Logistic(), 'use_GPU': use_GPU, 'debug': False, 'rng': np.random.RandomState(0) }, { 'max_epochs': max_epochs, 'CG_iter': 10 }, n_inputs=500, sig_len=200, plots=False) else: raise ValueError("Unknown profile function") if cprofile: p.disable() ps = pstats.Stats(p) ps.strip_dirs().sort_stats('time').print_stats(20) else: pycuda.driver.stop_profiler()
def prof_main(): from cProfile import Profile prof = Profile() prof.enable() fish = BasicGoFish(build_players()) fish.do_full_round() prof.disable() line_sep_char = '#' win_string = '{1} Winners: {0}'.format( ', '.join(['Player ' + p.name for p in fish.winner]), line_sep_char) sep = line_sep_char * len(win_string) print('\n{0}\n{1}\n{0}'.format(sep, win_string)) prof.print_stats() return fish
class BenchmarkThread(Thread): def __init__(self, thread_num, session, query, values, num_queries, profile): Thread.__init__(self) self.thread_num = thread_num self.session = session self.query = query self.values = values self.num_queries = num_queries self.profiler = Profile() if profile else None def start_profile(self): if self.profiler: self.profiler.enable() def finish_profile(self): if self.profiler: self.profiler.disable() self.profiler.dump_stats("profile-%d" % self.thread_num)
def some_func(foo, bar): """ :param foo: :param bar: :return: """ try: profile1 = Profile() profile1.enable() # to measure only a part of a function result = foo + bar profile1.disable() stats1 = pstats.Stats(profile) stats.print_stats() return result except Exception as exception: raise exception
def __init_profiler( profiler_arg: bool, logger: logging.Logger) -> Tuple[Optional[Profile], Optional[str]]: """ Initialise profiling environment by creating a cprofile.Profiler instance, a folder where pstats can be dumped Args: profiler_arg: the value of profiler argument passed when running the command logger: a logger instance Returns: If profiling enabled, a tuple of profiler instance and profiling directory where the stats files would be dumped, otherwise, a tuple of nulls """ if profiler_arg: logger.info('Profiling mode enabled') logger.debug('Creating & enabling profiler ...') profiler = Profile() profiler.enable() logger.debug('Profiler created.') profiling_dir = os.path.join( PROFILING_DIR, f'{datetime.utcnow().strftime("%Y%m%d_%H%M%S_%f")}_{generate_random_string(10)}', ) try: os.makedirs(profiling_dir) logger.debug('Profiling directory "%s" created', profiling_dir) except OSError as ex: if ex.errno != errno.EEXIST: raise logger.debug('Profiling directory "%s" already exists', profiling_dir) return profiler, profiling_dir logger.info('Profiling mode not enabled') return None, None
def compare_phi(): """ Problem 38 >>> a, b = compare_phi() >>> a > b True """ from cProfile import Profile from pstats import Stats pr_normal = Profile() pr_normal.enable() phi(10090) pr_normal.disable() time_normal = Stats(pr_normal).total_tt pr_improved = Profile() pr_improved.enable() phi_improved(10090) pr_improved.disable() time_improved = Stats(pr_improved).total_tt return time_normal, time_improved
def profile_rnn_calc_G(cprofile=True): """Run a profiler on the recurrent curvature calculation. :param bool cprofile: use True if profiling on the CPU, False if using the CUDA profiler """ inputs = np.random.randn(1024, 128, 1).astype(np.float32) targets = np.random.randn(1024, 128, 1).astype(np.float32) N = 128 rnn = hf.RNNet([1, N, 1], use_GPU=True) rnn.optimizer = hf.opt.HessianFree() # for struc_damping check rnn.cache_minibatch(inputs, targets) v = np.random.randn(rnn.W.size).astype(np.float32) for _ in range(2): # run it a few times to get rid of any startup overhead rnn.GPU_calc_G(v) if cprofile: start = time.time() p = Profile() p.enable() else: pycuda.driver.start_profiler() for _ in range(100): _ = rnn.GPU_calc_G(v) if cprofile: p.disable() print("time", time.time() - start) ps = pstats.Stats(p) ps.strip_dirs().sort_stats('time').print_stats(20) else: pycuda.driver.stop_profiler()
def test_ensemble(self,test_file): assert self.authenticated, 'Not authenticated!' # download a local copy of the ensemble self.logger.info('Creating local ensemble') local_ensemble = Ensemble(self.ensemble_res,api=self.api) # make the Fields object source = self.api.get_source(self.source_res) fields = Fields(source['object']['fields']) self.logger.info('Reading test data and generating predictions') true_labels = [] predict_labels = [] pr = Profile() pr.enable() with open(test_file) as fid: test_reader = csv.reader(fid) # skip the header line test_reader.next() for row in test_reader: row_list = [val for val in row] true_labels.append(row_list.pop()) instance = fields.pair(row_list) predict_labels.append(local_ensemble.predict(instance, by_name=False, method=1)) pr.disable() ps = Stats(pr) self.predict_time = ps.total_tt # eval_args = {'combiner':1} # evaluation = self.api.create_evaluation(self.ensemble_res,test_data,eval_args) # check_resource(evaluation['resource'],self.api.get_evaluation) # evaluation = self.api.get_evaluation(evaluation['resource']) # matrix = evaluation['object']['result']['model']['confusion_matrix'] # self.predict_time = evaluation['object']['status']['elapsed']/1000 if self.regression: self.results = (predict_labels,true_labels) else: self.results = make_confusion_matrix(true_labels,predict_labels)
def profile_calc_G(cprofile=True): """Run a profiler on the feedforward curvature calculation. :param bool cprofile: use True if profiling on the CPU, False if using the CUDA profiler """ inputs = np.random.randn(1024, 1).astype(np.float32) targets = np.random.randn(1024, 1).astype(np.float32) N = 1024 ff = hf.FFNet([1, N, N, 1], use_GPU=True) ff.cache_minibatch(inputs, targets) v = np.random.randn(ff.W.size).astype(np.float32) for _ in range(5): # run it a few times to get rid of any startup overhead ff.GPU_calc_G(v) if cprofile: start = time.time() p = Profile() p.enable() else: pycuda.driver.start_profiler() for _ in range(500): _ = ff.GPU_calc_G(v) if cprofile: p.disable() print("time", time.time() - start) ps = pstats.Stats(p) ps.strip_dirs().sort_stats('time').print_stats(20) else: pycuda.driver.stop_profiler()
def profile_dot(cprofile=True): """Run a profiler on the matrix multiplication kernel. :param bool cprofile: use True if profiling on the CPU, False if using the CUDA profiler """ N = 1024 a = np.random.randn(N, N).astype(np.float32) b = np.random.randn(N, N).astype(np.float32) a_gpu = gpuarray.to_gpu(a) b_gpu = gpuarray.to_gpu(b) c_gpu = gpuarray.zeros((N, N), np.float32) for _ in range(2): # run it a few times to get rid of any startup overhead hf.gpu.dot(a_gpu, b_gpu, out=c_gpu) if cprofile: start = time.time() p = Profile() p.enable() else: pycuda.autoinit.context.synchronize() pycuda.driver.start_profiler() for _ in range(100): hf.gpu.dot(a_gpu, b_gpu, out=c_gpu, transpose_a=True, transpose_b=True) c_gpu.get() if cprofile: p.disable() print("time", time.time() - start) ps = pstats.Stats(p) ps.strip_dirs().sort_stats('time').print_stats(20) else: pycuda.driver.stop_profiler()
class BenchmarkThread(Thread): def __init__(self, thread_num, session, query, values, num_queries, protocol_version, profile): Thread.__init__(self) self.thread_num = thread_num self.session = session self.query = query self.values = values self.num_queries = num_queries self.protocol_version = protocol_version self.profiler = Profile() if profile else None def start_profile(self): if self.profiler: self.profiler.enable() def run_query(self, key, **kwargs): return self.session.execute_async(self.query.format(key=key), **kwargs) def finish_profile(self): if self.profiler: self.profiler.disable() self.profiler.dump_stats("profile-%d" % self.thread_num)
def profile(func, max_epochs=1, use_GPU=False, cprofile=True): """Run a profiler on the code. :param str func: the demo function to be profiled (can be 'mnist' or 'integrator') :param int max_epochs: maximum number of iterations to run :param bool use_GPU: run optimization on GPU :param bool cprofile: if True then run the profiling on the CPU, otherwise use CUDA profiler """ if cprofile: p = Profile() p.enable() else: import pycuda pycuda.driver.start_profiler() if func == "mnist": mnist({'use_GPU': use_GPU, 'rng': np.random.RandomState(0)}, {'max_epochs': max_epochs, 'plotting': False, 'minibatch_size': 7500, 'CG_iter': 10}) elif func == "integrator": integrator({'shape': [1, 100, 1], 'layers': hf.nl.Logistic(), 'use_GPU': use_GPU, 'debug': False, 'rng': np.random.RandomState(0)}, {'max_epochs': max_epochs, 'CG_iter': 10}, n_inputs=500, sig_len=200, plots=False) else: raise ValueError("Unknown profile function") if cprofile: p.disable() ps = pstats.Stats(p) ps.strip_dirs().sort_stats('time').print_stats(20) else: pycuda.driver.stop_profiler()