def bench_publish(endpoint, appkey, channel, size, profile, ack=True): publisher = satori.rtm.connection.Connection(endpoint + '?appkey=' + appkey) publisher.start() message = binascii.hexlify(os.urandom(size // 2)).decode('ascii') print('Message size is {}'.format(len(message))) last_usage = [resource.getrusage(resource.RUSAGE_SELF)] print( 'Duration, s\tRate, msgs/s\tMax RSS, MB\tUser time, s\tSystem time, s') def report(duration, count): usage = resource.getrusage(resource.RUSAGE_SELF) maxrss = usage.ru_maxrss // 1024 if sys.platform == 'darwin': maxrss = maxrss // 1024 print('{0:2.2f}\t\t{1}\t\t{2}\t\t{3:2.2f}\t\t{4:2.2f}'.format( duration, int(count / duration), maxrss, usage.ru_utime - last_usage[0].ru_utime, usage.ru_stime - last_usage[0].ru_stime)) sys.stdout.flush() last_usage[0] = usage count = [0] def publish_without_ack(): publisher.publish(channel, message) count[0] += 1 def publish_with_ack(): def callback(ack): count[0] += 1 publisher.publish(channel, message, callback) publish = publish_with_ack if ack else publish_without_ack before = time.time() try: if profile: profiler = Profiler() profiler.start() while True: now = time.time() if now - before >= sampling_interval: report(now - before, count[0]) if profile: profiler.stop() print(profiler.output_text(unicode=True, color=True)) profiler = Profiler() profiler.start() count[0] = 0 before = time.time() publish() except KeyboardInterrupt: sys.exit(0)
def _wrapper(*args, **kwargs): _profiler = Profiler( sampling_interval_seconds ) if sampling_interval_seconds is not None else Profiler() _profiler.start() result = _func(*args, **kwargs) _profiler.stop() if print_to_console: print(_profiler.output_text(unicode=True, color=True)) if log_to_file: name = file_name if file_name is not None else _func.__name__ + ".profile" with open(f"{name}.html", "w") as f: f.write(_profiler.output_html()) return result
def profile_ctx(engine='pyinstrument'): """ A context manager which profiles the body of the with statement with the supplied profiling engine and returns the profiling object in a list. Arguments --------- engine: str The profiling engine, e.g. 'pyinstrument' or 'snakeviz' Returns ------- sessions: list A list containing the profiling session. """ if engine == 'pyinstrument': from pyinstrument import Profiler prof = Profiler() prof.start() elif engine == 'snakeviz': prof = Profile() prof.enable() elif engine is None: pass sessions = [] yield sessions if engine == 'pyinstrument': sessions.append(prof.stop()) elif engine == 'snakeviz': prof.disable() sessions.append(prof)
def wrapper(*args, **kwargs): profiler = Profiler() profiler.start() r = func(*args, **kwargs) profiler.stop() print profiler.output_text(color=True) return r
def __init__(self, conf, grass_use_file, args): self.conf = conf self.grass_use_file = grass_use_file if args.p: self.prof = Profiler() else: self.prof = None
def forward(self, x): profiler = Profiler() profiler.start() x = normalization(x) if x.shape[-1] > 3: pos = x[:, :, :3] feat = x[:, :, 3:] else: pos = x feat = None index_voxels, context_points, mask = self.selfvoxels(pos) pos, feat = self.sa_module1(pos, feat, index_voxels, context_points, mask) index_voxels, context_points, mask = self.selfvoxels(pos) pos, feat = self.sa_module2(pos, feat, index_voxels, context_points, mask) index_voxels, context_points, mask = self.selfvoxels(pos) h = self.sa_module3(pos, feat, index_voxels, context_points, mask) h = self.mlp1(h) h = self.bn1(h) h = F.relu(h) h = self.drop1(h) h = self.mlp2(h) h = self.bn2(h) h = F.relu(h) h = self.drop2(h) out = self.mlp_out(h) profiler.stop() print(profiler.output_text(unicode=True, color=True)) return out
def new_func(*args, **kwargs): import time from flask import has_request_context, g, current_app, request if (not has_request_context() or "profile" in request.args or not current_app.config.get("PROFILE") or current_app.config.get("PROFILE") == "False"): return func(*args, **kwargs) g.profiler = Profiler() if "profiler" not in g else g.profiler result = None with g.profiler: result = func(*args, **kwargs) # 写入文件 if g.profiler._last_session.duration * 1000 > timeout: time_format = time.strftime("%Y%m%d_%H:%M:%S", time.localtime()) file_path = f"./log/profile_{func.__name__}_{time_format}.txt" profile_file = open(file_path, "w") with profile_file: profile_file.write(g.profiler.output_text()) # 返回 return result
def profile(self, fct, **kwargs): """ Profiles function *fct*, calls it *repeat* times. @param fct function to profile (no argument) @param kwargs stores additional information about the profiling """ if self.module in ('pyinstrument', 'cProfile'): if self.module == 'pyinstrument': profiler = Profiler(interval=self.interval) start = profiler.start stop = profiler.stop else: profiler = c_Profile() start = profiler.enable stop = profiler.disable start() for _ in range(self.repeat): fct() stop() elif self.module == "profile": profiler = py_Profile() def lf(): for _ in range(self.repeat): fct() profiler.runcall(lf) else: raise ValueError( # pragma: no cover "Unknown profiler '{}'.".format(self.module)) self.profiled.append(profiler) self.kwargs.append(kwargs)
def instrument( cls_runner: tp.Type[Perf], pattern_func: str, timeline: bool = False, ) -> None: ''' Profile the `sf` function from the supplied class. ''' runner = cls_runner() for name in runner.iter_function_names(pattern_func): f = getattr(runner, name) profiler = Profiler(interval=0.0001) # default is 0.001, 1 ms if timeline: profiler.start() f() profiler.stop() else: profiler.start() for _ in range(runner.NUMBER): f() profiler.stop() print( profiler.output_text(unicode=True, color=True, timeline=timeline, show_all=True))
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') # creating random file for each usage of this function f= open("/home/"+str(random.randint(1,1100))+".html","w+") profiler = Profiler(use_signal=False) profiler.start() n = int(req.params.get('n', 15)) if not n: try: req_body = req.get_json() except ValueError: pass else: name = req_body.get('n') if n: output = str(fibnonci_approach(n)) # stopping profiler and writing it to random file generated earlier profiler.stop() f.write(profiler.output_html()) return func.HttpResponse("Fibnonci of " + str(n) + " using regular approach:" + output) else: return func.HttpResponse( "Please pass a name on the query string or in the request body", status_code=400 )
def test_get_symmetries(): """Check that the function correctly creates six new reflected boards and policy vectors, one for each axis of symmetry. """ game = BloomsGame(size=4, score_target=15) board = game.getInitBoard() # Place stones board.place_stone(position=(3, 1), colour=1) board.place_stone(position=(5, 1), colour=2) board.place_stone(position=(3, 5), colour=3) board.place_stone(position=(1, 5), colour=4) # board.visualise(show_coords=True, title="Original") # Create a dummy policy vector pi = np.random.random_sample(game.getActionSize()) profiler = Profiler() profiler.start() symmetrical_states = game.getSymmetries(board, pi) profiler.stop() print(profiler.output_text(unicode=True, color=True)) assert len(symmetrical_states) == 24 assert symmetrical_states[1][1][37] == pi[52] assert symmetrical_states[1][1][614] == pi[1277] assert all( [sum(x[1]) == pytest.approx(sum(pi)) for x in symmetrical_states])
def _report_from_filters( self, n_clicks, dd_name, input_name, *filter_values ): if n_clicks == 0: raise ValueError("no true report request, only toggle") stime = time.time() name = input_name for elem in dash.callback_context.triggered: if ElemIds.is_dd(elem["prop_id"]): name = dd_name logger.info("start filtering") if self._timing: logger.info("starting time profiling") profiler = Profiler() profiler.start() filtered_ddf = self._filter_ddf(filter_values) logger.info("calculating elements") elems = self.get_report_elems(filtered_ddf) logger.info(f"{len(elems)} elements returned") logger.info("parsing elements to output values") outputs = self._parse_report_elems_to_outputs(elems, name) logger.info(f"{len(outputs)} outputs parsed") logger.info("report done", runtime=round(time.time() - stime, 3)) if self._timing: profiler.stop() log_profiler(profiler, logger, 20) return (name, name, *outputs)
def forward(self, x): profiler = Profiler() profiler.start() if x.shape[-1] > 3: pos = x[:, :, :3] feat = x[:, :, 3:] else: pos = x feat = None pos, feat = self.sa_msg_module1(pos, feat) pos, feat = self.sa_msg_module2(pos, feat) h = self.sa_module3(pos, feat) h = self.mlp1(h) h = self.bn1(h) h = F.relu(h) h = self.drop1(h) h = self.mlp2(h) h = self.bn2(h) h = F.relu(h) h = self.drop2(h) out = self.mlp_out(h) profiler.stop() print(profiler.output_text(unicode=True, color=True)) return out
def profile(output_format='html', output_filename=None, html_open=False): """Profiles a block of code with Pyinstrument. Intended to be used in a `with` statement. Parameters ---------- output_format : {'html', 'text'}, optional Shows the result either as text or in an HTML file. If :code:`output_format = 'html'`, the file is saved according to the :code:`output_filename` parameter. The default is 'html'. output_filename : str, optional Only taken into account if :code:`output_format = 'html'`. If not given (default), the html output is saved to the same directory the caller resides. The name of the html file is the same as that of the caller. html_open : bool, optional Only taken into account if :code:`output_format = 'html'`. If True, the generated HTML file is opened in the default browser. The default is False. Yields ------ Notes ----- In the implementation, we move two levels up in the stack frame, one for exiting the context manager and one for exiting this generator. This assumes that :meth:`profile` was called as a context manager. As a good practice, provide the :code:`output_filename` input argument. Examples -------- Measure the time needed to generate 1 million uniformly distributed random numbers. >>> import random >>> with profile('html') as p: ... for _ in range(1000000): ... rand_num = random.uniform(1, 2.2) """ # If required, guess the path where the results will be saved if output_format == 'html' and not output_filename: caller = inspect.currentframe().f_back.f_back.f_locals['__file__'] output_filename = os.path.splitext(caller)[0] + '.html' # Start the profiler profiler = Profiler() profiler.start() # Give back the execution to the caller function yield # Finish profiling and show the results profiler.stop() if output_format == 'html': if html_open: HTMLRenderer().open_in_browser(profiler.last_session, output_filename=output_filename) else: with codecs.open(output_filename, 'w', 'utf-8') as f: f.write(HTMLRenderer().render(profiler.last_session)) elif output_format == 'text': print(profiler.output_text(unicode=True, color=True, show_all=True))
def test_context_manager(): with Profiler() as profiler: long_function_a() long_function_b() frame = profiler.first_interesting_frame() assert frame.function == 'test_context_manager' assert len(frame.children) == 2
def wrapper(*args, **kwargs): from pyinstrument import Profiler profiler = Profiler() profiler.start() result = func(*args, **kwargs) profiler.stop() print(profiler.output_text(unicode=True, color=True)) return result
def test_context_manager(): with Profiler() as profiler: long_function_a() long_function_b() frame = profiler.last_session.root_frame() assert frame.function == 'test_context_manager' assert len(frame.children) == 2
def train_models( self, partitions_path, graphs_path, init_lr=1e-4, train_batch_size=32, valid_batch_size=32, epoch_batch=5, epochs=100, checkpoint_path="checkpoint/aux_30_epoch_", # This version only looks at race + age (not gender) ): with open(partitions_path, "rb") as fp: # Unpickling set_partitions = pickle.load(fp) # Training Auxiliary Models! for i in range(0, int(epochs / epoch_batch)): data_generator_aux = UtkFaceDataGeneratorAuxOneModel( self.dataset_folder_name, set_partitions[i], self.dataset_dict, num_classes=self.num_classes) aux_train_idx, aux_valid_idx, _ = data_generator_aux.generate_split_indexes( ) train_images_collection, train_status_collection = data_generator_aux.pre_generate_images( aux_train_idx, batch_size=train_batch_size) valid_images_collection, valid_status_collection = data_generator_aux.pre_generate_images( aux_valid_idx, batch_size=valid_batch_size) aux_train_gen = data_generator_aux.generate_images( is_training=True, images_collection=train_images_collection, status_collection=train_status_collection) aux_valid_gen = data_generator_aux.generate_images( is_training=True, images_collection=valid_images_collection, status_collection=valid_status_collection) aux_model = self.build_model() es = EarlyStopping(monitor='val_loss', mode='min', patience=10) profiler = Profiler() profiler.start() history = aux_model.fit( aux_train_gen, steps_per_epoch=len(aux_train_idx) // train_batch_size, epochs=(i + 1) * 5, validation_data=aux_valid_gen, validation_steps=len(aux_valid_idx) // valid_batch_size, callbacks=[es]) profiler.stop() profiler.print(show_all=True) aux_model.save(str(checkpoint_path) + "_" + str((i + 1) * 5)) y = history.history['val_loss'] plt.plot([i for i in range(len(y))], history.history['val_loss']) plt.title("Auxiliary Model Validation Loss - {} Epochs".format( (i + 1) * 5)) plt.savefig(graphs_path / "aux_30_epoch_val_loss_{}".format( (i + 1) * 5))
def process_request(self, request): profile_dir = getattr(settings, 'PYINSTRUMENT_PROFILE_DIR', None) if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT', 'profile') in request.GET or profile_dir: profiler = Profiler() profiler.start() request.profiler = profiler
async def main(): p = Profiler(async_mode='disabled') with p: for _ in range(10_000): await asyncio.gather( example(random.randint(10, 1000)), example(random.randint(10, 10000)), example(random.randint(10, 1000)), )
def process_request(self, request): if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT', 'profile') in request.GET: self.profiler = Profiler() try: self.profiler.start() except NotMainThreadError: raise NotMainThreadError(not_main_thread_message) self.profiler = None
def train_model(self, init_lr=1e-4, train_batch_size=16, valid_batch_size=16, epoch_batch=5, epochs=100, checkpoint_dir="checkpoint/base_epochs_"): opt = Adam(lr=init_lr, decay=init_lr / epochs) self.model.compile(optimizer=opt, loss={ 'age_output': 'mse', 'race_output': 'categorical_crossentropy', 'gender_output': 'binary_crossentropy' }, loss_weights={ 'age_output': 4., 'race_output': 1.5, 'gender_output': 0.1 }, metrics={ 'age_output': 'mae', 'race_output': 'accuracy', 'gender_output': 'accuracy' }) for i in range(int(epochs / epoch_batch)): current_checkpoint = checkpoint_dir / "base_epochs_{}".format( str((i + 1) * epoch_batch)) if i != 0: self.model = load_model( checkpoint_dir / "base_epochs_{}".format(str((i) * epoch_batch))) print(len(self.train_idx), len(self.valid_idx)) train_images_collection, train_status_collection = self.data_generator.pre_generate_images( self.train_idx, batch_size=train_batch_size) valid_images_collection, valid_status_collection = self.data_generator.pre_generate_images( self.valid_idx, batch_size=valid_batch_size) train_gen = self.data_generator.generate_images( is_training=True, images_collection=train_images_collection, status_collection=train_status_collection) valid_gen = self.data_generator.generate_images( is_training=True, images_collection=valid_images_collection, status_collection=valid_status_collection) profiler = Profiler() profiler.start() history = self.model.fit_generator( train_gen, steps_per_epoch=len(self.train_idx) // train_batch_size, # epochs=epoch_batch, epochs=1, validation_data=valid_gen, validation_steps=len(self.valid_idx) // valid_batch_size) profiler.stop() profiler.print(show_all=True)
def profile(): profiler = Profiler() profiler.start() exit_code = runtests.run_tests_all(parallel=False) profiler.stop() print(profiler.output_text(unicode=False, color=True)) return exit_code
def setUp(self): """Create a test dataset.""" self.profiler = Profiler() dataset_file = '../tests/data/tomo_setup.pickle.lzma' with lzma.open(dataset_file, 'rb') as file: [ self.data, self.theta, self.original, ] = pickle.load(file)
def setUp(self): self.profiler = Profiler() dataset_file = '../tests/data/lamino_setup.pickle.lzma' with lzma.open(dataset_file, 'rb') as file: [ self.data, self.original, self.theta, self.tilt, ] = pickle.load(file)
def pytinstrument_decorator(*args, **kwargs): # Create profiler instance before function is called. profiler = Profiler() profiler.start() # Call function. func_return = func(*args, **kwargs) # Stop profiler and print results after function is called. profiler.stop() print(profiler.output_text(unicode=False, color=True)) return func_return
def test_profiler_doesnt_overflow_on_large_call_stacks(): profiler = Profiler() profiler.start() # give 170 frames of leeway for the test runner and for pyinstrument to do its work. recursion_depth = sys.getrecursionlimit() - 170 recurse(recursion_depth) profiler.stop() print(profiler.output_text())
def deep_profiler_session(): profiler = Profiler() profiler.start() # give 120 frames for pyinstrument to do its work. recursion_depth = sys.getrecursionlimit() - current_stack_depth() - 120 recurse(recursion_depth) profiler.stop() return profiler.last_session
def test_profiler_doesnt_overflow_on_large_call_stacks(): profiler = Profiler() profiler.start() # give ourselves 150 frames of leeway to do our work. recursion_depth = sys.getrecursionlimit() - 150 recurse(recursion_depth) profiler.stop() print(profiler.output_text())
def wrapper(*args, **kwargs): profiler = None if CONFIG.profiler_enable: profiler = Profiler() profiler.start() try: return f(*args, **kwargs) finally: if profiler is not None: profiler.stop() print(profiler.output_text(unicode=True, color=True))