def test_get_symmetries(): """Check that the function correctly creates six new reflected boards and policy vectors, one for each axis of symmetry. """ game = BloomsGame(size=4, score_target=15) board = game.getInitBoard() # Place stones board.place_stone(position=(3, 1), colour=1) board.place_stone(position=(5, 1), colour=2) board.place_stone(position=(3, 5), colour=3) board.place_stone(position=(1, 5), colour=4) # board.visualise(show_coords=True, title="Original") # Create a dummy policy vector pi = np.random.random_sample(game.getActionSize()) profiler = Profiler() profiler.start() symmetrical_states = game.getSymmetries(board, pi) profiler.stop() print(profiler.output_text(unicode=True, color=True)) assert len(symmetrical_states) == 24 assert symmetrical_states[1][1][37] == pi[52] assert symmetrical_states[1][1][614] == pi[1277] assert all( [sum(x[1]) == pytest.approx(sum(pi)) for x in symmetrical_states])
class ProfilerMiddleware(object): def __init__(self): self.profiler = None def process_request(self, request): if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT', 'profile') in request.GET: self.profiler = Profiler() try: self.profiler.start() except NotMainThreadError: raise NotMainThreadError(not_main_thread_message) self.profiler = None def process_response(self, request, response): if self.profiler: try: self.profiler.stop() return HttpResponse(self.profiler.output_html()) except NotMainThreadError: raise NotMainThreadError(not_main_thread_message) finally: self.profiler = None else: return response
class BenchmarkTomo(unittest.TestCase): """Run benchmarks for tomography reconstruction.""" def setUp(self): """Create a test dataset.""" self.profiler = Profiler() dataset_file = '../tests/data/tomo_setup.pickle.lzma' with lzma.open(dataset_file, 'rb') as file: [ self.data, self.theta, self.original, ] = pickle.load(file) @unittest.skip('Demonstrate skipped tests.') def test_never(self): """Never run this test.""" pass def test_cgrad(self): """Use pyinstrument to benchmark tomo.grad on one core.""" logging.disable(logging.WARNING) result = {'obj': np.zeros(self.original.shape, dtype=np.complex64)} self.profiler.start() for i in range(50): result = tike.tomo.reconstruct( **result, theta=self.theta, integrals=self.data, algorithm='cgrad', num_iter=1, ) self.profiler.stop() print('\n') print(self.profiler.output_text(unicode=True, color=True))
def _report_from_filters( self, n_clicks, dd_name, input_name, *filter_values ): if n_clicks == 0: raise ValueError("no true report request, only toggle") stime = time.time() name = input_name for elem in dash.callback_context.triggered: if ElemIds.is_dd(elem["prop_id"]): name = dd_name logger.info("start filtering") if self._timing: logger.info("starting time profiling") profiler = Profiler() profiler.start() filtered_ddf = self._filter_ddf(filter_values) logger.info("calculating elements") elems = self.get_report_elems(filtered_ddf) logger.info(f"{len(elems)} elements returned") logger.info("parsing elements to output values") outputs = self._parse_report_elems_to_outputs(elems, name) logger.info(f"{len(outputs)} outputs parsed") logger.info("report done", runtime=round(time.time() - stime, 3)) if self._timing: profiler.stop() log_profiler(profiler, logger, 20) return (name, name, *outputs)
def instrument( cls_runner: tp.Type[Perf], pattern_func: str, timeline: bool = False, ) -> None: ''' Profile the `sf` function from the supplied class. ''' runner = cls_runner() for name in runner.iter_function_names(pattern_func): f = getattr(runner, name) profiler = Profiler(interval=0.0001) # default is 0.001, 1 ms if timeline: profiler.start() f() profiler.stop() else: profiler.start() for _ in range(runner.NUMBER): f() profiler.stop() print( profiler.output_text(unicode=True, color=True, timeline=timeline, show_all=True))
def forward(self, x): profiler = Profiler() profiler.start() x = normalization(x) if x.shape[-1] > 3: pos = x[:, :, :3] feat = x[:, :, 3:] else: pos = x feat = None index_voxels, context_points, mask = self.selfvoxels(pos) pos, feat = self.sa_module1(pos, feat, index_voxels, context_points, mask) index_voxels, context_points, mask = self.selfvoxels(pos) pos, feat = self.sa_module2(pos, feat, index_voxels, context_points, mask) index_voxels, context_points, mask = self.selfvoxels(pos) h = self.sa_module3(pos, feat, index_voxels, context_points, mask) h = self.mlp1(h) h = self.bn1(h) h = F.relu(h) h = self.drop1(h) h = self.mlp2(h) h = self.bn2(h) h = F.relu(h) h = self.drop2(h) out = self.mlp_out(h) profiler.stop() print(profiler.output_text(unicode=True, color=True)) return out
def main(req: func.HttpRequest) -> func.HttpResponse: logging.info('Python HTTP trigger function processed a request.') # creating random file for each usage of this function f= open("/home/"+str(random.randint(1,1100))+".html","w+") profiler = Profiler(use_signal=False) profiler.start() n = int(req.params.get('n', 15)) if not n: try: req_body = req.get_json() except ValueError: pass else: name = req_body.get('n') if n: output = str(fibnonci_approach(n)) # stopping profiler and writing it to random file generated earlier profiler.stop() f.write(profiler.output_html()) return func.HttpResponse("Fibnonci of " + str(n) + " using regular approach:" + output) else: return func.HttpResponse( "Please pass a name on the query string or in the request body", status_code=400 )
class ManagerProfilePyinstrument(ManagerProfile): def __init__(self, sync: bool): self.sync = sync self.profiler = Profiler() def start(self): self.profiler.start() return nullcontext() def stop_and_write(self, path_profile: str, is_docker: bool, api: str, render_browser: bool = False): self.profiler.stop() mode = "sync" if self.sync else "async" filename = f"pyinstrument_profile_{mode}_{api}.html" if not is_docker: output_html = self.profiler.output_html() self._write_output_file(path_profile, output_html, filename=filename) if render_browser: self.profiler.open_in_browser() print(self.profiler.output_text(unicode=True, color=True)) def _write_output_file(self, path_profile: str, output_html: str, filename: str): output_html_path = self._prepare_output_path(path_profile, filename) with open(output_html_path, "w") as file: file.write(output_html)
def wrapper(*args, **kwargs): profiler = Profiler() profiler.start() r = func(*args, **kwargs) profiler.stop() print profiler.output_text(color=True) return r
def forward(self, x): profiler = Profiler() profiler.start() if x.shape[-1] > 3: pos = x[:, :, :3] feat = x[:, :, 3:] else: pos = x feat = None pos, feat = self.sa_msg_module1(pos, feat) pos, feat = self.sa_msg_module2(pos, feat) h = self.sa_module3(pos, feat) h = self.mlp1(h) h = self.bn1(h) h = F.relu(h) h = self.drop1(h) h = self.mlp2(h) h = self.bn2(h) h = F.relu(h) h = self.drop2(h) out = self.mlp_out(h) profiler.stop() print(profiler.output_text(unicode=True, color=True)) return out
def profile(output_format='html', output_filename=None, html_open=False): """Profiles a block of code with Pyinstrument. Intended to be used in a `with` statement. Parameters ---------- output_format : {'html', 'text'}, optional Shows the result either as text or in an HTML file. If :code:`output_format = 'html'`, the file is saved according to the :code:`output_filename` parameter. The default is 'html'. output_filename : str, optional Only taken into account if :code:`output_format = 'html'`. If not given (default), the html output is saved to the same directory the caller resides. The name of the html file is the same as that of the caller. html_open : bool, optional Only taken into account if :code:`output_format = 'html'`. If True, the generated HTML file is opened in the default browser. The default is False. Yields ------ Notes ----- In the implementation, we move two levels up in the stack frame, one for exiting the context manager and one for exiting this generator. This assumes that :meth:`profile` was called as a context manager. As a good practice, provide the :code:`output_filename` input argument. Examples -------- Measure the time needed to generate 1 million uniformly distributed random numbers. >>> import random >>> with profile('html') as p: ... for _ in range(1000000): ... rand_num = random.uniform(1, 2.2) """ # If required, guess the path where the results will be saved if output_format == 'html' and not output_filename: caller = inspect.currentframe().f_back.f_back.f_locals['__file__'] output_filename = os.path.splitext(caller)[0] + '.html' # Start the profiler profiler = Profiler() profiler.start() # Give back the execution to the caller function yield # Finish profiling and show the results profiler.stop() if output_format == 'html': if html_open: HTMLRenderer().open_in_browser(profiler.last_session, output_filename=output_filename) else: with codecs.open(output_filename, 'w', 'utf-8') as f: f.write(HTMLRenderer().render(profiler.last_session)) elif output_format == 'text': print(profiler.output_text(unicode=True, color=True, show_all=True))
def wrapper(*args, **kwargs): from pyinstrument import Profiler profiler = Profiler() profiler.start() result = func(*args, **kwargs) profiler.stop() print(profiler.output_text(unicode=True, color=True)) return result
def train_models( self, partitions_path, graphs_path, init_lr=1e-4, train_batch_size=32, valid_batch_size=32, epoch_batch=5, epochs=100, checkpoint_path="checkpoint/aux_30_epoch_", # This version only looks at race + age (not gender) ): with open(partitions_path, "rb") as fp: # Unpickling set_partitions = pickle.load(fp) # Training Auxiliary Models! for i in range(0, int(epochs / epoch_batch)): data_generator_aux = UtkFaceDataGeneratorAuxOneModel( self.dataset_folder_name, set_partitions[i], self.dataset_dict, num_classes=self.num_classes) aux_train_idx, aux_valid_idx, _ = data_generator_aux.generate_split_indexes( ) train_images_collection, train_status_collection = data_generator_aux.pre_generate_images( aux_train_idx, batch_size=train_batch_size) valid_images_collection, valid_status_collection = data_generator_aux.pre_generate_images( aux_valid_idx, batch_size=valid_batch_size) aux_train_gen = data_generator_aux.generate_images( is_training=True, images_collection=train_images_collection, status_collection=train_status_collection) aux_valid_gen = data_generator_aux.generate_images( is_training=True, images_collection=valid_images_collection, status_collection=valid_status_collection) aux_model = self.build_model() es = EarlyStopping(monitor='val_loss', mode='min', patience=10) profiler = Profiler() profiler.start() history = aux_model.fit( aux_train_gen, steps_per_epoch=len(aux_train_idx) // train_batch_size, epochs=(i + 1) * 5, validation_data=aux_valid_gen, validation_steps=len(aux_valid_idx) // valid_batch_size, callbacks=[es]) profiler.stop() profiler.print(show_all=True) aux_model.save(str(checkpoint_path) + "_" + str((i + 1) * 5)) y = history.history['val_loss'] plt.plot([i for i in range(len(y))], history.history['val_loss']) plt.title("Auxiliary Model Validation Loss - {} Epochs".format( (i + 1) * 5)) plt.savefig(graphs_path / "aux_30_epoch_val_loss_{}".format( (i + 1) * 5))
def train_model(self, init_lr=1e-4, train_batch_size=16, valid_batch_size=16, epoch_batch=5, epochs=100, checkpoint_dir="checkpoint/base_epochs_"): opt = Adam(lr=init_lr, decay=init_lr / epochs) self.model.compile(optimizer=opt, loss={ 'age_output': 'mse', 'race_output': 'categorical_crossentropy', 'gender_output': 'binary_crossentropy' }, loss_weights={ 'age_output': 4., 'race_output': 1.5, 'gender_output': 0.1 }, metrics={ 'age_output': 'mae', 'race_output': 'accuracy', 'gender_output': 'accuracy' }) for i in range(int(epochs / epoch_batch)): current_checkpoint = checkpoint_dir / "base_epochs_{}".format( str((i + 1) * epoch_batch)) if i != 0: self.model = load_model( checkpoint_dir / "base_epochs_{}".format(str((i) * epoch_batch))) print(len(self.train_idx), len(self.valid_idx)) train_images_collection, train_status_collection = self.data_generator.pre_generate_images( self.train_idx, batch_size=train_batch_size) valid_images_collection, valid_status_collection = self.data_generator.pre_generate_images( self.valid_idx, batch_size=valid_batch_size) train_gen = self.data_generator.generate_images( is_training=True, images_collection=train_images_collection, status_collection=train_status_collection) valid_gen = self.data_generator.generate_images( is_training=True, images_collection=valid_images_collection, status_collection=valid_status_collection) profiler = Profiler() profiler.start() history = self.model.fit_generator( train_gen, steps_per_epoch=len(self.train_idx) // train_batch_size, # epochs=epoch_batch, epochs=1, validation_data=valid_gen, validation_steps=len(self.valid_idx) // valid_batch_size) profiler.stop() profiler.print(show_all=True)
def bench_publish(endpoint, appkey, channel, size, profile, ack=True): publisher = satori.rtm.connection.Connection(endpoint + '?appkey=' + appkey) publisher.start() message = binascii.hexlify(os.urandom(size // 2)).decode('ascii') print('Message size is {}'.format(len(message))) last_usage = [resource.getrusage(resource.RUSAGE_SELF)] print( 'Duration, s\tRate, msgs/s\tMax RSS, MB\tUser time, s\tSystem time, s') def report(duration, count): usage = resource.getrusage(resource.RUSAGE_SELF) maxrss = usage.ru_maxrss // 1024 if sys.platform == 'darwin': maxrss = maxrss // 1024 print('{0:2.2f}\t\t{1}\t\t{2}\t\t{3:2.2f}\t\t{4:2.2f}'.format( duration, int(count / duration), maxrss, usage.ru_utime - last_usage[0].ru_utime, usage.ru_stime - last_usage[0].ru_stime)) sys.stdout.flush() last_usage[0] = usage count = [0] def publish_without_ack(): publisher.publish(channel, message) count[0] += 1 def publish_with_ack(): def callback(ack): count[0] += 1 publisher.publish(channel, message, callback) publish = publish_with_ack if ack else publish_without_ack before = time.time() try: if profile: profiler = Profiler() profiler.start() while True: now = time.time() if now - before >= sampling_interval: report(now - before, count[0]) if profile: profiler.stop() print(profiler.output_text(unicode=True, color=True)) profiler = Profiler() profiler.start() count[0] = 0 before = time.time() publish() except KeyboardInterrupt: sys.exit(0)
def profile(): profiler = Profiler() profiler.start() exit_code = runtests.run_tests_all(parallel=False) profiler.stop() print(profiler.output_text(unicode=False, color=True)) return exit_code
def pytinstrument_decorator(*args, **kwargs): # Create profiler instance before function is called. profiler = Profiler() profiler.start() # Call function. func_return = func(*args, **kwargs) # Stop profiler and print results after function is called. profiler.stop() print(profiler.output_text(unicode=False, color=True)) return func_return
def test_profiler_doesnt_overflow_on_large_call_stacks(): profiler = Profiler() profiler.start() # give 170 frames of leeway for the test runner and for pyinstrument to do its work. recursion_depth = sys.getrecursionlimit() - 170 recurse(recursion_depth) profiler.stop() print(profiler.output_text())
def deep_profiler_session(): profiler = Profiler() profiler.start() # give 120 frames for pyinstrument to do its work. recursion_depth = sys.getrecursionlimit() - current_stack_depth() - 120 recurse(recursion_depth) profiler.stop() return profiler.last_session
def test_profiler_doesnt_overflow_on_large_call_stacks(): profiler = Profiler() profiler.start() # give ourselves 150 frames of leeway to do our work. recursion_depth = sys.getrecursionlimit() - 150 recurse(recursion_depth) profiler.stop() print(profiler.output_text())
def main(): usage = "usage: %prog [-h] [-o output_file_path] scriptfile [arg] ..." parser = OptionParser(usage=usage) parser.allow_interspersed_args = False parser.add_option('', '--html', dest="output_html", action='store_true', help="output HTML instead of text", default=False) parser.add_option('-o', '--outfile', dest="outfile", action='store', help="save stats to <outfile>", default=None) if not sys.argv[1:]: parser.print_usage() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args if len(args) > 0: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with open(progname, 'rb') as fp: code = compile(fp.read(), progname, 'exec') globs = { '__file__': progname, '__name__': '__main__', '__package__': None, } profiler = Profiler() profiler.start() try: exec code in globs, None except SystemExit, KeyboardInterrupt: pass profiler.stop() if options.outfile: f = codecs.open(options.outfile, 'w', 'utf-8') unicode = True color = False else: f = sys.stdout unicode = stdout_supports_unicode() color = stdout_supports_color() if options.output_html: f.write(profiler.output_html()) else: f.write(profiler.output_text(unicode=unicode, color=color)) f.close()
def wrapper(*args, **kwargs): profiler = None if CONFIG.profiler_enable: profiler = Profiler() profiler.start() try: return f(*args, **kwargs) finally: if profiler is not None: profiler.stop() print(profiler.output_text(unicode=True, color=True))
def main(): profiler = Profiler() sim = Simulation() sim.setup() profiler.start() for i in range(100): sim.update() profiler.stop() print(profiler.output_text(unicode=True, color=True))
def profiler_session(session: str) -> None: "invoke code with sampling profiler" has_profiler = profiler_enable() if has_profiler: profiler = Profiler(interval=profiler_interval()) profiler.start() try: yield finally: if has_profiler: profiler.stop() profiler_produce_report(profiler, session)
def instrument(filename="results"): profiler = Profiler() directory = Path("profiling/results/") try: profiler.start() yield finally: profiler.stop() directory.mkdir(parents=True, exist_ok=True) path = os.path.join(directory, f"{filename}.html") with open(path, "w") as fs: fs.write(profiler.output_html(timeline=True))
def decorated_view(*args, **kwargs): profiler = Profiler() profiler.start() result = func(*args, **kwargs) profiler.stop() print(profiler.output_text(unicode=True, color=True)) return result
def reverse(self, num_steps=1, width=5, height=5, msb_bits=10): old_boards_positions = [ (1000, LifeBoard(board=-1 * np.ones(self.board.shape, dtype=np.int32))) ] # levels - ranges of each tilling levels = self.build_levels(width, height) max_old_boards, max_tiling, indices = 3, 200, None # TODO - find good start (high density - 1's number) debug_performance, type_performance = False, 1 if debug_performance: if type_performance == 0: pr = cProfile.Profile() pr.enable() else: profiler = Profiler() profiler.start() for level in levels: # print("level:", level) tic_level = time.time() interior_str = self.interior_str([(1, 1), (1, 1)], level) old_boards_tilings = self.read_tilings(interior_str, width, height, max_tiling) # TODO - soften selection (another parameter) # TODO - selection based on probability # TODO - selection based on real fit for delta old_boards_positions = old_boards_positions[:max_old_boards] #print("first selection: ", len(old_boards_tilings), len(old_boards_positions)) #TODO - constrint new to current old_boards_results = [ ob_pos[1].position(ob_tiling, level, constraint=self) for ob_pos in old_boards_positions for ob_tiling in old_boards_tilings ] #print("stage 1(fit): ", time.time() - tic_level) #indices = np.argsort([result[0] for result in old_boards_results])[:max_old_boards] old_boards_positions = sorted(old_boards_results, key=lambda pos: pos[0]) #print("stage 2(sort): ", time.time() - tic_level) # print("second selection: ", len(old_boards_positions),old_boards_positions[0][0]) if debug_performance: if type_performance == 0: pr.disable() pr.print_stats(sort="time") else: profiler.stop() print(profiler.output_text(unicode=True, color=True)) # print(len(old_boards_positions),old_boards_positions[0][0]) return old_boards_positions[0][1]
def _wrapper(*args, **kwargs): _profiler = Profiler( sampling_interval_seconds ) if sampling_interval_seconds is not None else Profiler() _profiler.start() result = _func(*args, **kwargs) _profiler.stop() if print_to_console: print(_profiler.output_text(unicode=True, color=True)) if log_to_file: name = file_name if file_name is not None else _func.__name__ + ".profile" with open(f"{name}.html", "w") as f: f.write(_profiler.output_html()) return result
def instrument(cls: tp.Type[PerfTest], function: str = 'sf') -> None: ''' Profile the `sf` function from the supplied class. ''' f = getattr(cls, function) profiler = Profiler() profiler.start() for _ in range(cls.NUMBER): f() profiler.stop() print(profiler.output_text(unicode=True, color=True))
def test_profiler_retains_multiple_calls(): profiler = Profiler() profiler.start() long_function_a() long_function_b() long_function_a() long_function_b() profiler.stop() print(profiler.output_text()) frame = profiler.last_session.root_frame() assert frame.function == 'test_profiler_retains_multiple_calls' assert len(frame.children) == 4
def profile_ctx(engine='pyinstrument'): """ A context manager which profiles the body of the with statement with the supplied profiling engine and returns the profiling object in a list. Arguments --------- engine: str The profiling engine, e.g. 'pyinstrument' or 'snakeviz' Returns ------- sessions: list A list containing the profiling session. """ if engine == 'pyinstrument': from pyinstrument import Profiler prof = Profiler() prof.start() elif engine == 'snakeviz': prof = Profile() prof.enable() elif engine is None: pass sessions = [] yield sessions if engine == 'pyinstrument': sessions.append(prof.stop()) elif engine == 'snakeviz': prof.disable() sessions.append(prof)
def test_profiler_retains_multiple_calls(): profiler = Profiler() profiler.start() long_function_a() long_function_b() long_function_a() long_function_b() profiler.stop() print(profiler.output_text()) frame = profiler.first_interesting_frame() assert frame.function == 'test_profiler_retains_multiple_calls' assert len(frame.children) == 4
def main(): usage = "usage: %prog [-h] [[-o output_file_path] scriptfile [arg] ...] | [ -i infile ]" parser = OptionParser(usage=usage) parser.allow_interspersed_args = False parser.add_option('', '--html', dest="output_html", action='store_true', help="output HTML instead of text", default=False) parser.add_option('', '--json', dest="output_json", action='store_true', help="output raw JSON dump instead of text or HTML", default=False) parser.add_option('-o', '--outfile', dest="outfile", action='store', help="save stats to <outfile>", default=None) parser.add_option('-i', '--infile', dest="infile", action='store', help="load stats from JSON file <infile>", default=None) if not sys.argv[1:]: parser.print_usage() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args if len(args) > 0: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with open(progname, 'rb') as fp: code = compile(fp.read(), progname, 'exec') globs = { '__file__': progname, '__name__': '__main__', '__package__': None, } profiler = Profiler() profiler.start() try: exec code in globs, None except SystemExit, KeyboardInterrupt: pass profiler.stop() write_output( options, profiler )
def run_profiling(args): lprofiler = LineProfiler() monitor_fuctions = [api.problem.submit_key, api.problem.get_unlocked_pids, api.problem.get_solved_pids, api.problem.get_all_problems, api.problem.get_solved_problems, api.stats.get_score, api.cache.memoize, api.autogen.grade_problem_instance, api.autogen.get_problem_instance, api.autogen.get_number_of_instances] for func in monitor_fuctions: lprofiler.add_function(func) lprofiler.enable() if args.stack: profiler = Profiler(use_signal=False) profiler.start() for func, a, kw in operations: func(*a, **kw) if args.stack: profiler.stop() lprofiler.disable() if args.print: print(profiler.output_text(unicode=True, color=True)) lprofiler.print_stats() output = open(args.output, "w") if args.stack: output.write(profiler.output_text(unicode=True)) if args.output_html is not None: output_html = open(args.output_html, "w") output_html.write(profiler.output_html()) output_html.close() print("Wrote test info to " + args.output_html) lprofiler.print_stats(output) output.close() print("Wrote test info to " + args.output)
def test_collapses_multiple_calls_by_default(): profiler = Profiler() profiler.start() long_function_a() long_function_b() long_function_a() long_function_b() profiler.stop() text_output = profiler.output_text() # output should be something like: # 1.513 test_collapses_multiple_calls_by_default test/test_profiler.py:25 # |- 0.507 long_function_a test/test_profiler.py:17 # |- 0.503 long_function_b test/test_profiler.py:20 assert text_output.count('test_collapses_multiple_calls_by_default') == 1 assert text_output.count('long_function_a') == 1 assert text_output.count('long_function_b') == 1
def test_two_functions(): profiler = Profiler() profiler.start() long_function_a() long_function_b() profiler.stop() print(profiler.output_text()) frame = profiler.last_session.root_frame() assert frame.function == 'test_two_functions' assert len(frame.children) == 2 frame_b, frame_a = sorted(frame.children, key=lambda f: f.time(), reverse=True) assert frame_a.function == 'long_function_a' assert frame_b.function == 'long_function_b' assert 0.2 < frame_a.time() < 0.3 assert 0.45 < frame_b.time() < 0.55
def main(): """Command-line utility for using (and testing) s3 utility methods.""" logging.basicConfig(level=logging.DEBUG) arg_parser = argparse.ArgumentParser(description='Perform obfuscation of forum .mongo dump files.') arg_parser.add_argument( 'input', help='Read mongo files from this location.', ) arg_parser.add_argument( '-o', '--output', help='Write obfuscated mongo files to this location in the local file system.', default=None ) arg_parser.add_argument( '-u', '--userinfo', help='For events, read a custom user-info file from the local fs that contains username, email, user-id, fullname.', default=None ) arg_parser.add_argument( '--log-context', help='characters on each side of match', type=int, default=50, ) ##################### # Flags to indicate what to obfuscate. ##################### arg_parser.add_argument( '--forum', help='Read in and obfuscate forum posts.', action='store_true', ) arg_parser.add_argument( '--wiki', help='Read in and obfuscate wiki documents.', action='store_true', ) arg_parser.add_argument( '--courseware', help='Read in and obfuscate courseware_studentmodule records.', action='store_true', ) arg_parser.add_argument( '--event', help='Read in and obfuscate events.', action='store_true', ) ##################### # Various flags to indicate what to look for. ##################### arg_parser.add_argument( '--phone', help='Extract phone numbers', action='store_true', ) arg_parser.add_argument( '--possible-phone', help='Extract phone numbers', action='store_true', ) arg_parser.add_argument( '--email', help='Extract email addresses', action='store_true', ) arg_parser.add_argument( '--phone-context', help='Extract phone number context', action='store_true', ) arg_parser.add_argument( '--email-context', help='Extract email address context', action='store_true', ) arg_parser.add_argument( '--name-context', help='Extract name context', action='store_true', ) arg_parser.add_argument( '--facebook', help='Extract facebook urls', action='store_true', ) arg_parser.add_argument( '--username', help='Extract username', action='store_true', ) arg_parser.add_argument( '--fullname', help='Extract fullname.', action='store_true', ) arg_parser.add_argument( '--userid', help='Extract user-id.', action='store_true', ) arg_parser.add_argument( '--skip-post', help='Skip performing filtering on event.POST entries.', action='store_true', ) arg_parser.add_argument( '--pyinstrument', help='Profile the run and write the output to stderr', action='store_true' ) args = arg_parser.parse_args() kwargs = vars(args) profiler = None if args.pyinstrument: profiler = Profiler() # or Profiler(use_signal=False), see below profiler.start() try: obfuscator = BulkObfuscator(**kwargs) obfuscator.obfuscate_directory(args.input, args.output) finally: if profiler: profiler.stop() print >>sys.stderr, profiler.output_text(unicode=True, color=True)
def main(): usage = ("usage: pyinstrument [options] scriptfile [arg] ...") version_string = 'pyinstrument {v}, on Python {pyv[0]}.{pyv[1]}.{pyv[2]}'.format( v=pyinstrument.__version__, pyv=sys.version_info, ) parser = optparse.OptionParser(usage=usage, version=version_string) parser.allow_interspersed_args = False def dash_m_callback(option, opt, value, parser): parser.values.module_name = value # everything after the -m argument should be passed to that module parser.values.module_args = parser.rargs + parser.largs parser.rargs[:] = [] parser.largs[:] = [] parser.add_option('', '--load-prev', dest='load_prev', action='store', metavar='ID', help="Instead of running a script, load a previous report") parser.add_option('-m', '', dest='module_name', action='callback', callback=dash_m_callback, type="str", help="run library module as a script, like 'python -m module'") parser.add_option('-o', '--outfile', dest="outfile", action='store', help="save to <outfile>", default=None) parser.add_option('-r', '--renderer', dest='renderer', action='store', type='string', help=("how the report should be rendered. One of: 'text', 'html', 'json', or python " "import path to a renderer class"), default='text') parser.add_option('', '--html', dest="output_html", action='store_true', help=optparse.SUPPRESS_HELP, default=False) # deprecated shortcut for --renderer=html parser.add_option('-t', '--timeline', dest='timeline', action='store_true', help="render as a timeline - preserve ordering and don't condense repeated calls") parser.add_option('', '--hide', dest='hide_fnmatch', action='store', metavar='EXPR', help=("glob-style pattern matching the file paths whose frames to hide. Defaults to " "'*{sep}lib{sep}*'.").format(sep=os.sep), default='*{sep}lib{sep}*'.format(sep=os.sep)) parser.add_option('', '--hide-regex', dest='hide_regex', action='store', metavar='REGEX', help=("regex matching the file paths whose frames to hide. Useful if --hide doesn't give " "enough control.")) parser.add_option('', '--show', dest='show_fnmatch', action='store', metavar='EXPR', help=("glob-style pattern matching the file paths whose frames to " "show, regardless of --hide or --hide-regex. For example, use " "--show '*/<library>/*' to show frames within a library that " "would otherwise be hidden.")) parser.add_option('', '--show-regex', dest='show_regex', action='store', metavar='REGEX', help=("regex matching the file paths whose frames to always show. " "Useful if --show doesn't give enough control.")) parser.add_option('', '--show-all', dest='show_all', action='store_true', help="show everything", default=False) parser.add_option('', '--unicode', dest='unicode', action='store_true', help='(text renderer only) force unicode text output') parser.add_option('', '--no-unicode', dest='unicode', action='store_false', help='(text renderer only) force ascii text output') parser.add_option('', '--color', dest='color', action='store_true', help='(text renderer only) force ansi color text output') parser.add_option('', '--no-color', dest='color', action='store_false', help='(text renderer only) force no color text output') if not sys.argv[1:]: parser.print_help() sys.exit(2) options, args = parser.parse_args() if args == [] and options.module_name is None and options.load_prev is None: parser.print_help() sys.exit(2) if not options.hide_regex: options.hide_regex = fnmatch.translate(options.hide_fnmatch) if not options.show_regex and options.show_fnmatch: options.show_regex = fnmatch.translate(options.show_fnmatch) if options.show_all: options.show_regex = r'.*' if options.load_prev: session = load_report(options.load_prev) else: if options.module_name is not None: sys.argv[:] = [options.module_name] + options.module_args code = "run_module(modname, run_name='__main__')" globs = { 'run_module': runpy.run_module, 'modname': options.module_name } else: sys.argv[:] = args progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with open(progname, 'rb') as fp: code = compile(fp.read(), progname, 'exec') globs = { '__file__': progname, '__name__': '__main__', '__package__': None, } profiler = Profiler() profiler.start() try: exec_(code, globs, None) except (SystemExit, KeyboardInterrupt): pass profiler.stop() session = profiler.last_session if options.output_html: options.renderer = 'html' output_to_temp_file = (options.renderer == 'html' and not options.outfile and file_is_a_tty(sys.stdout)) if options.outfile: f = codecs.open(options.outfile, 'w', 'utf-8') should_close_f_after_writing = True elif not output_to_temp_file: if PY2: f = codecs.getwriter('utf-8')(sys.stdout) else: f = sys.stdout should_close_f_after_writing = False renderer_kwargs = {'processor_options': { 'hide_regex': options.hide_regex, 'show_regex': options.show_regex, }} if options.timeline is not None: renderer_kwargs['timeline'] = options.timeline if options.renderer == 'text': unicode_override = options.unicode != None color_override = options.color != None unicode = options.unicode if unicode_override else file_supports_unicode(f) color = options.color if color_override else file_supports_color(f) renderer_kwargs.update({'unicode': unicode, 'color': color}) renderer_class = get_renderer_class(options.renderer) renderer = renderer_class(**renderer_kwargs) # remove this frame from the trace renderer.processors.append(remove_first_pyinstrument_frame_processor) if output_to_temp_file: output_filename = renderer.open_in_browser(session) print('stdout is a terminal, so saved profile output to %s' % output_filename) else: f.write(renderer.render(session)) if should_close_f_after_writing: f.close() if options.renderer == 'text': _, report_identifier = save_report(session) print('To view this report with different options, run:') print(' pyinstrument --load-prev %s [options]' % report_identifier) print('')
def main(): usage = ("usage: python -m pyinstrument [options] scriptfile [arg] ...") parser = OptionParser(usage=usage) parser.allow_interspersed_args = False parser.add_option('', '--html', dest="output_html", action='store_true', help="output HTML instead of text", default=False) parser.add_option('-o', '--outfile', dest="outfile", action='store', help="save report to <outfile>", default=None) parser.add_option('', '--unicode', dest='unicode', action='store_true', help='force unicode text output') parser.add_option('', '--no-unicode', dest='unicode', action='store_false', help='force ascii text output') parser.add_option('', '--color', dest='color', action='store_true', help='force ansi color text output') parser.add_option('', '--no-color', dest='color', action='store_false', help='force no color text output') if not sys.argv[1:]: parser.print_help() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args if len(args) > 0: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with open(progname, 'rb') as fp: code = compile(fp.read(), progname, 'exec') globs = { '__file__': progname, '__name__': '__main__', '__package__': None, } try: profiler = Profiler() except SignalUnavailableError: profiler = Profiler(use_signal=False) profiler.start() try: exec_(code, globs, None) except (SystemExit, KeyboardInterrupt): pass profiler.stop() if options.outfile: f = codecs.open(options.outfile, 'w', 'utf-8') else: f = sys.stdout unicode_override = options.unicode != None color_override = options.color != None unicode = options.unicode if unicode_override else file_supports_unicode(f) color = options.color if color_override else file_supports_color(f) if options.output_html: f.write(profiler.output_html()) else: f.write(profiler.output_text(unicode=unicode, color=color)) f.close() else: parser.print_usage() return parser
class OpenAssessmentTest(WebAppTest): """ UI-level acceptance tests for Open Assessment. """ TEST_COURSE_ID = "course-v1:edx+ORA203+course" PROBLEM_LOCATIONS = { 'staff_only': u'courses/{test_course_id}/courseware/' u'61944efb38a349edb140c762c7419b50/415c3ee1b7d04b58a1887a6fe82b31d6/'.format(test_course_id=TEST_COURSE_ID), 'self_only': u'courses/{test_course_id}/courseware/' u'a4dfec19cf9b4a6fb5b18be6ccd9cecc/338a4affb58a45459629e0566291381e/'.format(test_course_id=TEST_COURSE_ID), 'peer_only': u'courses/{test_course_id}/courseware/' u'a4dfec19cf9b4a6fb5b18be6ccd9cecc/417e47b2663a4f79b62dba20b21628c8/'.format(test_course_id=TEST_COURSE_ID), 'student_training': u'courses/{test_course_id}/courseware/' u'676026889c884ac1827688750871c825/5663e9b038434636977a4226d668fe02/'.format(test_course_id=TEST_COURSE_ID), 'file_upload': u'courses/{test_course_id}/courseware/' u'57a3f9d51d424f6cb922f0d69cba868d/bb563abc989340d8806920902f267ca3/'.format(test_course_id=TEST_COURSE_ID), 'full_workflow_staff_override': u'courses/{test_course_id}/courseware/' u'676026889c884ac1827688750871c825/181ea9ff144c4766be44eb8cb360e34f/'.format(test_course_id=TEST_COURSE_ID), 'full_workflow_staff_required': u'courses/{test_course_id}/courseware/' u'8d9584d242b44343bc270ea5ef04ab03/0b0dcc728abe45138c650732af178afb/'.format(test_course_id=TEST_COURSE_ID), } SUBMISSION = u"This is a test submission." LATEX_SUBMISSION = u"[mathjaxinline]( \int_{0}^{1}xdx )[/mathjaxinline]" OPTIONS_SELECTED = [1, 2] STAFF_OVERRIDE_OPTIONS_SELECTED = [0, 1] STAFF_OVERRIDE_SCORE = 1 STAFF_GRADE_EXISTS = "COMPLETE" STAFF_OVERRIDE_LEARNER_STEPS_NOT_COMPLETE = "YOU MUST COMPLETE THE STEPS ABOVE TO VIEW YOUR GRADE" STAFF_AREA_SCORE = "Final grade: {} out of 8" STAFF_OVERRIDE_STAFF_AREA_NOT_COMPLETE = "The problem has not been completed." EXPECTED_SCORE = 6 STUDENT_TRAINING_OPTIONS = [ [1, 2], [0, 2] ] TEST_PASSWORD = "******" def setUp(self, problem_type, staff=False): """ Configure page objects to test Open Assessment. Args: problem_type (str): The type of problem being tested, used to choose which part of the course to load. staff (bool): If True, runs the test with a staff user (defaults to False). """ super(OpenAssessmentTest, self).setUp() if PROFILING_ENABLED: self.profiler = Profiler(use_signal=False) self.profiler.start() self.problem_loc = self.PROBLEM_LOCATIONS[problem_type] self.auto_auth_page = AutoAuthPage(self.browser, course_id=self.TEST_COURSE_ID, staff=staff) self.submission_page = SubmissionPage(self.browser, self.problem_loc) self.self_asmnt_page = AssessmentPage('self-assessment', self.browser, self.problem_loc) self.peer_asmnt_page = AssessmentPage('peer-assessment', self.browser, self.problem_loc) self.student_training_page = AssessmentPage('student-training', self.browser, self.problem_loc) self.staff_asmnt_page = AssessmentPage('staff-assessment', self.browser, self.problem_loc) self.grade_page = GradePage(self.browser, self.problem_loc) def log_to_file(self): with open('{}-profile.log'.format(self.id()), 'w') as f: f.write(self.profiler.output_text()) def tearDown(self): if PROFILING_ENABLED: self.profiler.stop() self.log_to_file() def login_user(self, learner, email): """ Logs in an already existing user. Args: learner (str): the username of the user. email (str): email address of the user. """ auto_auth_page = AutoAuthPage( self.browser, email=email, password=self.TEST_PASSWORD, username=learner, course_id=self.TEST_COURSE_ID, staff=True ) auto_auth_page.visit() def do_self_assessment(self): """ Creates a user, submits a self assessment, verifies the grade, and returns the username of the learner for which the self assessment was submitted. """ self.auto_auth_page.visit() username, _ = self.auto_auth_page.get_username_and_email() self.submission_page.visit().submit_response(self.SUBMISSION) self.assertTrue(self.submission_page.has_submitted) # Submit a self-assessment self.submit_self_assessment(self.OPTIONS_SELECTED) # Verify the grade self.assertEqual(self.EXPECTED_SCORE, self.grade_page.wait_for_page().score) return username def submit_self_assessment(self, options=OPTIONS_SELECTED): """ Submit a self assessment for the currently logged in student. Do not verify grade. Args: options: the options to select for the self assessment (will use OPTIONS_SELECTED if not specified) """ self.self_asmnt_page.wait_for_page().wait_for_response() self.assertIn(self.SUBMISSION, self.self_asmnt_page.response_text) self.self_asmnt_page.assess("self", options).wait_for_complete() self.assertTrue(self.self_asmnt_page.is_complete) def _verify_staff_grade_section(self, expected_status, expected_message_title): """ Verifies the expected status and message text in the Staff Grade section (as shown to the learner). """ self.staff_asmnt_page.wait_for_page() self.assertEqual("Staff Grade", self.staff_asmnt_page.label) self.staff_asmnt_page.verify_status_value(expected_status) self.assertEqual(expected_message_title, self.staff_asmnt_page.message_title) def do_training(self): """ Complete two training examples, satisfying the requirements. """ for example_num, options_selected in enumerate(self.STUDENT_TRAINING_OPTIONS): if example_num > 0: try: self.student_training_page.wait_for_num_completed(example_num) except BrokenPromise: msg = "Did not complete at least {num} student training example(s).".format(num=example_num) self.fail(msg) self.student_training_page.wait_for_page().wait_for_response().assess("training", options_selected) # Check that we've completed student training try: self.student_training_page.wait_for_complete() except BrokenPromise: self.fail("Student training was not marked complete.") def do_peer_assessment(self, count=1, options=OPTIONS_SELECTED): """ Does the specified number of peer assessments. Args: count: the number of assessments that must be completed (defaults to 1) options: the options to use (defaults to OPTIONS_SELECTED) """ self.peer_asmnt_page.visit() for count_assessed in range(1, count + 1): self.peer_asmnt_page.wait_for_page().wait_for_response().assess("peer", options) self.peer_asmnt_page.wait_for_num_completed(count_assessed) def do_staff_override(self, username, final_score=STAFF_AREA_SCORE.format(STAFF_OVERRIDE_SCORE)): """ Complete a staff assessment (grade override). Args: username: the learner to grade final_score: the expected final score as shown in the staff area (defaults to the staff override score value) """ self.staff_area_page.visit() self.staff_area_page.show_learner(username) self.staff_area_page.expand_learner_report_sections() self.staff_area_page.staff_assess(self.STAFF_OVERRIDE_OPTIONS_SELECTED, "override") self.staff_area_page.verify_learner_final_score(final_score) def do_staff_assessment(self, number_to_assess=0, options_selected=OPTIONS_SELECTED): """ Use staff tools to assess available responses. Args: number_to_assess: the number of submissions to assess. If not provided (or 0), will grade all available submissions. """ self.staff_area_page.visit() self.staff_area_page.click_staff_toolbar_button("staff-grading") # Get the counts before checking out a submission for assessment. start_numbers = self.staff_area_page.available_checked_out_numbers # Check out a submission. self.staff_area_page.expand_staff_grading_section() # Checked out number should increase, ungraded decrease. ungraded = start_numbers[0]-1 checked_out = start_numbers[1]+1 self.staff_area_page.verify_available_checked_out_numbers((ungraded, checked_out)) assessed = 0 while number_to_assess == 0 or assessed < number_to_assess: continue_after = False if number_to_assess-1 == assessed else ungraded > 0 self.staff_area_page.staff_assess(options_selected, "full-grade", continue_after) assessed += 1 if not continue_after: self.staff_area_page.verify_available_checked_out_numbers((ungraded, checked_out-1)) break else: ungraded -=1 self.staff_area_page.verify_available_checked_out_numbers((ungraded, checked_out)) def refresh_page(self): """ Helper method that waits for "unsaved changes" warnings to clear before refreshing the page. """ EmptyPromise( lambda: self.browser.execute_script("return window.onbeforeunload === null"), "Unsubmitted changes exist on page." ).fulfill() self.browser.refresh()
def observe(config_file, profile=False, raise_queue_empty=True): if profile: try: from pyinstrument import Profiler except ImportError: print "Error importing pyinstrument" profile = False ztf_config = ZTFConfiguration("../sims/{}".format(config_file)) # load config parameters into local variables run_name = ztf_config.config["run_name"] start_time = ztf_config.config["start_time"] weather_year = ztf_config.config["weather_year"] if weather_year == "None": weather_year = None survey_duration = ztf_config.config["survey_duration_days"] * u.day block_programs = ztf_config.config["block_programs"] observing_programs = ztf_config.build_observing_programs() if profile: if survey_duration > 1.0 * u.day: print ("Don't profile long runs: 25% overhead") profile = False else: profiler = Profiler() profiler.start() survey_start_time = Time(start_time, scale="utc", location=P48_loc) tel = ZTFStateMachine( current_time=survey_start_time, historical_observability_year=weather_year, logfile="../sims/{}_log.txt".format(run_name), ) # set up QueueManager Q = GreedyQueueManager(block_programs=block_programs) for op in observing_programs: Q.add_observing_program(op) # initialize nightly field requests (Tom Barlow function) Q.assign_nightly_requests(tel.current_state_dict()) # initialize sqlite history log = ObsLogger(run_name, tel.current_time) current_night_mjd = np.floor(tel.current_time.mjd) while tel.current_time < (survey_start_time + survey_duration): # check if it is a new night and reload queue with new requests if np.floor(tel.current_time.mjd) > current_night_mjd: log.prev_obs = None Q.assign_nightly_requests(tel.current_state_dict()) current_night_mjd = np.floor(tel.current_time.mjd) if tel.check_if_ready(): current_state = tel.current_state_dict() # get coords try: next_obs = Q.next_obs(current_state) # TODO: debugging check... assert next_obs["request_id"] in Q.queue.index except QueueEmptyError: if not raise_queue_empty: tel.logger.info("Queue empty! Waiting...") log.prev_obs = None tel.wait() continue else: raise QueueEmptyError("Queue is empty") # try to change filters, if needed if next_obs["target_filter_id"] != current_state["current_filter_id"]: if not tel.start_filter_change(next_obs["target_filter_id"]): tel.logger.info("Filter change failure! Waiting...") log.prev_obs = None tel.wait() continue # try to slew to the next target if not tel.start_slew(coord.SkyCoord(next_obs["target_ra"] * u.deg, next_obs["target_dec"] * u.deg)): tel.set_cant_observe() # TODO: log the failure # "missed history": http://ops2.lsst.org/docs/current/architecture.html#output-tables tel.logger.info( "Failure slewing to {}, {}! Waiting...".format( next_obs["target_ra"] * u.deg, next_obs["target_dec"] * u.deg ) ) log.prev_obs = None tel.wait() continue # try to expose if not tel.start_exposing(): tel.set_cant_observe() tel.logger.info("Exposure failure! Waiting...") log.prev_obs = None tel.wait() continue else: # exposure completed successfully. now # a) store exposure information in pointing history sqlite db current_state = tel.current_state_dict() log.log_pointing(current_state, next_obs) # b) update Fields Q.fields.mark_field_observed(next_obs, current_state) # c) remove completed request_id from the pool and the queue # TODO: debugging check assert next_obs["request_id"] in Q.queue.index Q.remove_requests(next_obs["request_id"]) else: log.prev_obs = None tel.set_cant_observe() tel.wait() if profile: profiler.stop() print profiler.output_text(unicode=True, color=True) with open("../sims/{}_profile.txt".format(run_name), "w") as f: f.write(profiler.output_text())
def main(): usage = ("usage: pyinstrument [options] scriptfile [arg] ...") parser = OptionParser(usage=usage) parser.allow_interspersed_args = False parser.add_option('', '--setprofile', dest='setprofile', action='store_true', help='run in setprofile mode, instead of signal mode', default=False) parser.add_option('', '--html', dest="output_html", action='store_true', help="output HTML instead of text", default=False) parser.add_option('-o', '--outfile', dest="outfile", action='store', help="save report to <outfile>", default=None) parser.add_option('', '--unicode', dest='unicode', action='store_true', help='force unicode text output') parser.add_option('', '--no-unicode', dest='unicode', action='store_false', help='force ascii text output') parser.add_option('', '--color', dest='color', action='store_true', help='force ansi color text output') parser.add_option('', '--no-color', dest='color', action='store_false', help='force no color text output') if not sys.argv[1:]: parser.print_help() sys.exit(2) (options, args) = parser.parse_args() sys.argv[:] = args if len(args) > 0: progname = args[0] sys.path.insert(0, os.path.dirname(progname)) with open(progname, 'rb') as fp: code = compile(fp.read(), progname, 'exec') globs = { '__file__': progname, '__name__': '__main__', '__package__': None, } try: profiler = Profiler(use_signal=not options.setprofile) except SignalUnavailableError: profiler = Profiler(use_signal=False) profiler.start() try: exec_(code, globs, None) except IOError as e: import errno if e.errno == errno.EINTR: print( 'Failed to run program due to interrupted system system call.\n' 'This happens because pyinstrument is sending OS signals to the running\n' 'process to interrupt it. If your program has long-running syscalls this\n' 'can cause a problem.\n' '\n' 'You can avoid this error by running in \'setprofile\' mode. Do this by\n' 'passing \'--setprofile\' when calling pyinstrument at the command-line.\n' '\n' 'For more information, see\n' 'https://github.com/joerick/pyinstrument/issues/16\n' ) raise except (SystemExit, KeyboardInterrupt): pass profiler.stop() if options.outfile: f = codecs.open(options.outfile, 'w', 'utf-8') else: f = sys.stdout unicode_override = options.unicode != None color_override = options.color != None unicode = options.unicode if unicode_override else file_supports_unicode(f) color = options.color if color_override else file_supports_color(f) if options.output_html: f.write(profiler.output_html()) else: f.write(profiler.output_text(unicode=unicode, color=color)) f.close() else: parser.print_usage() return parser
def observe(run_name=run_name, start_time='2018-01-01 04:00:00', weather_year=None, survey_duration=1 * u.hour): if profile: profiler = Profiler() profiler.start() survey_start_time = Time(start_time, scale='utc', location=P48_loc) tel = ZTFStateMachine( current_time=survey_start_time, historical_observability_year=weather_year) # set up QueueManager Q = GreedyQueueManager() # set up Observing Programs #CollabOP = CollaborationObservingProgram() # Q.add_observing_program(CollabOP) MSIPOP = MSIPObservingProgram( Q.fields.select_field_ids(dec_range=[-30, 90], grid_id=0)) MSIPOP.observing_time_fraction = 1.0 Q.add_observing_program(MSIPOP) #CaltechOP = CaltechObservingProgram() # Q.add_observing_program(CaltechOP) # initialize nightly field requests (Tom Barlow function) Q.assign_nightly_requests(tel.current_state_dict()) # temporary loading to test things # Q.rp.add_requests(1, # Q.fields.fields[ # Q.fields.select_fields(dec_range=[-30,90])].index, 2, # 'no_cadence',{}) # initialize sqlite history log = ObsLogger(run_name, tel.current_time) log.create_pointing_log(clobber=True) while tel.current_time < (survey_start_time + survey_duration): # TODO: reload queue with new requests on update interval (nightly if tel.check_if_ready(): current_state = tel.current_state_dict() # get coords next_obs = Q.next_obs(current_state) # TODO: filter change, if needed if not tel.start_slew(coord.SkyCoord(next_obs['target_ra'] * u.deg, next_obs['target_dec'] * u.deg)): tel.set_cant_observe() # TODO: log the failure # "missed history": http://ops2.lsst.org/docs/current/architecture.html#output-tables log.prev_obs = None tel.wait() continue if not tel.start_exposing(): tel.set_cant_observe() # TODO: log the failure log.prev_obs = None tel.wait() continue else: # exposure completed successfully. now # a) store exposure information in pointing history sqlite db current_state = tel.current_state_dict() log.log_pointing(current_state, next_obs) # b) update Fields Q.fields.mark_field_observed(next_obs, current_state) # c) remove completed request_id Q.rp.remove_requests(next_obs['request_id']) else: tel.set_cant_observe() tel.wait() if profile: profiler.stop() print profiler.output_text(unicode=True, color=True)
from pyinstrument import Profiler import numpy as np import pyamg profiler = Profiler() profiler.start() n = int(1e3) A = pyamg.gallery.poisson((n, n), format='csr') b = np.random.rand(A.shape[0]) ml = pyamg.smoothed_aggregation_solver(A, max_coarse=10) res = [] x = ml.solve(b, accel='cg', residuals=res) print(len(res)) profiler.stop() print(profiler.output_text(unicode=True, color=True))
from pyinstrument import Profiler from platform import platform p = Profiler() p.start() def func(): fd = open('/dev/urandom', 'rb') data = fd.read(1024*1024) func() # this failed on ubuntu 12.04 platform() p.stop() print(p.output_text()) with open('ioerror_out.html', 'w') as f: f.write(p.output_html())