def experiment(variant): cuda = True from gym.envs.mujoco import HalfCheetahEnv from mujoco_torch.core.bridge import MjCudaRender R = 84 env = HalfCheetahEnv() c = Convnet(6, output_activation=torch.tanh, input_channels=3) if cuda: c.cuda() gt.stamp("start") for i in range(100): img = env.sim.render(R, R, device_id=1) gt.stamp("warmstart") for i in gt.timed_for(range(1000)): env.step(np.random.rand(6)) gt.stamp('step') img = env.sim.render(R, R, device_id=1) gt.stamp('render') x = np_to_var(img) if cuda: x = x.cuda() torch.cuda.synchronize() gt.stamp('transfer') # cv2.imshow("img", img) # cv2.waitKey(1) gt.stamp("end") print(img) print(gt.report(include_itrs=False))
def main(): gt.reset() tries = 0 # starting with just 9x9. sudoku = [] # 1D list of all sudoku's elements. while len(sudoku) < 81: tries += 1 num_list = list(range(1, 10)) num = random.choice(num_list) while inRow(sudoku, num) or inCol(sudoku, num) or inSquare( sudoku, num): try: num_list.remove(num) num = random.choice(num_list) except IndexError: sudoku = [] sudoku.append(num) PrintSudoku(sudoku) # substring in report with total time total_time = float(gt.report()[91:97].split()[0]) total_time *= 1000 # convert seconds ot milliseconds print("tries: " + str(tries)) print("total time: " + str(total_time) + " ms") # sudoku attempts per millisecond print("Attempts per ms: " + str(tries / total_time))
def experiment(variant): root = 0 E = 20 R = 84 U = 6 cuda = True envs = [] for e in range(E): env = HalfCheetahEnv() envs.append(env) c = Convnet(6, output_activation=torch.tanh, input_channels=3) if cuda: c.cuda() # viewer = mujoco_py.MjRenderContextOffscreen(env.sim, device_id=1) # env.sim.add_render_context(viewer) def step(i, stamp=True): imgs = [] if i % 100 == 0: for e in envs: e.reset() for e in envs: img = e.sim.render(R, R, device_id=0).transpose() imgs.append(img) gt.stamp('render') if stamp else 0 imgs = np.array(imgs) torch_img = np_to_var(imgs) if cuda: torch_img = torch_img.cuda() torch.cuda.synchronize() gt.stamp('transfer') if stamp else 0 u = get_numpy(c.forward(torch_img).cpu()) torch.cuda.synchronize() gt.stamp('forward') if stamp else 0 for i, e in enumerate(envs): e.step(u[i, :]) gt.stamp('step') if stamp else 0 for i in range(10): step(i, False) gt.stamp('start') for i in gt.timed_for(range(100)): step(i) gt.stamp('end') print(gt.report(include_itrs=False, format_options=dict(itr_num_width=10)))
def experiment(variant): from gym.envs.mujoco import HalfCheetahEnv from mujoco_torch.core.bridge import MjCudaRender renderer = MjCudaRender(84, 84) env = HalfCheetahEnv() gt.stamp("start") for i in range(100): tensor, img = renderer.get_cuda_tensor(env.sim, False) gt.stamp("warmstart") for i in range(1000): env.step(np.random.rand(6)) tensor, img = renderer.get_cuda_tensor(env.sim, True) x = np_to_var(img).cuda() torch.cuda.synchronize() # cv2.imshow("img", img) # cv2.waitKey(1) gt.stamp("end") print(img) print(gt.report())
def experiment(variant): from gym.envs.mujoco import HalfCheetahEnv from mujoco_torch.core.bridge import MjCudaRender renderer = MjCudaRender(84, 84) env = HalfCheetahEnv() gt.stamp("start") for i in range(100): tensor, img = renderer.get_cuda_tensor(env.sim, False) gt.stamp("warmstart") for i in gt.timed_for(range(1000)): env.step(np.random.rand(6)) gt.stamp('step') tensor, img = renderer.get_cuda_tensor(env.sim, False) gt.stamp('render') # cv2.imshow("img", img) # cv2.waitKey(1) gt.stamp("end") print(img) print(gt.report(include_itrs=False))
import gtimer as gt import time time.sleep(0.1) gt.stamp('first') time.sleep(0.3) gt.stamp('second') print gt.report()
def train_epoch(epoch): print('\nTRAINING : Epoch ' + str(epoch)) model.train() losses = [] logs = [] last_time = time.time() metrics = Metrics(tok2i, i2tok, field=TRG) for i, batch in enumerate(trainloader): # -- Actual Training gt.reset() gt.stamp("load_data") oracle = Oracle(batch.trg[0].detach(), model.n_classes, tok2i, i2tok, **oracle_flags) gt.stamp("create_oracle") max_steps = 2*batch.trg[0].detach().ne(tok2i[constants.PAD_WORD]).sum(1).max()+1 scores, samples, p_oracle = model.forward(xs=batch.src, oracle=oracle, max_steps=max_steps, num_samples=len(batch), return_p_oracle=True) gt.stamp("forward") loss = loss_fn(scores, samples, p_oracle, end_idx=tok2i['<end>'], **loss_flags) gt.stamp("loss") optimizer.zero_grad() loss.backward() clip_grad_norm_(model.parameters(), args.max_norm) optimizer.step() gt.stamp("backward") losses.append(loss.item()) # -- Report metrics every `print_every` batches. if i % args.print_every == 0: # Only compute training metrics once here for efficiency. metrics.update(scores, samples, (batch.trg[0], None), kind='train') gt.stamp("metrics.update") # Training report computed over the last `print_every` batches. ms = metrics.report('train') ms['train/loss'] = round(np.mean(losses), 2) logs.append('{0} ; loss {1} ; sentence/s {2} ; {3} train {4} '.format( i+1, round(np.mean(losses), 2), int(len(losses) * args.batch_size / (time.time() - last_time)), args.eval_metric, ms['train/%s' % args.eval_metric], )) args.logstep += 1 last_time = time.time() losses = [] metrics.reset() # -- Validation report with a single batch. metrics.reset() model.eval() batch = next(iter(validloader)) scores, samples = predict_batch(batch) model.train() metrics.update(scores, samples, (batch.trg[0], None)) vms = metrics.report('valid_batch') logs[-1] = logs[-1] + metrics.log(vms, 'valid_batch', ['bleu', 'avg_span', 'f1', 'em', 'depth_score']) metrics.reset() print_samples(samples, (batch.trg[0], None), n=len(batch)) gt.stamp("validation_batch") log_tensorboard(ms, step=args.logstep) log_tensorboard(vms, step=args.logstep) print(logs[-1]) print(gt.report(include_itrs=False, format_options={'itr_name_width': 30})) # -- Checkpointing if i % args.save_every == 0: print('saving checkpoint at epoch {0} batch {1}'.format(epoch, i)) print(os.path.join(args.log_directory, args.expr_name + '.checkpoint')) torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'optimizer_param': args.optimizer, 'loss': loss.item() }, os.path.join(args.log_directory, args.expr_name + '.checkpoint')) model_config['longest_label'] = model.longest_label with open(os.path.join(args.log_directory, 'model_config.json'), 'w') as f: json.dump(model_config, f) print('end : epoch {0} '.format(epoch)) log_tensorboard({'lr': optimizer.param_groups[0]['lr']}, step=args.logstep)
import gtimer as gt import time time.sleep(0.1) gt.stamp('first') loop = gt.timed_loop('named_loop') x = 0 while x < 3: next(loop) time.sleep(0.1) x += 1 gt.stamp('loop') loop.exit() time.sleep(0.1) gt.stamp('second') print gt.report(include_itrs=False)
def _train(self, env, policy): self._init_training(env, policy) with self._sess.as_default(): observation = env.reset() policy.reset() itr = 0 path_length = 0 path_return = 0 gt.rename_root('online algo') gt.reset() gt.set_def_unique(False) for epoch in gt.timed_for(range(self._n_epochs), save_itrs=True): logger.push_prefix('Epoch #%d | ' % epoch) for t in range(self._epoch_length): # Sample next action and state. action, _ = policy.get_action(observation) gt.stamp('train: get actions') action.squeeze() if self._render: env.render() next_ob, raw_reward, terminal, info = env.step(action) reward = raw_reward * self._scale_reward path_length += 1 path_return += reward gt.stamp('train: simulation') # Add experience to replay pool. self._pool.add_sample(observation, action, reward, terminal, False) should_reset = (terminal or path_length >= self._max_path_length) if should_reset: # noinspection PyTypeChecker self._pool.add_sample(next_ob, np.zeros_like(action), np.zeros_like(reward), np.zeros_like(terminal), True) observation = env.reset() policy.reset() path_length = 0 path_return = 0 else: observation = next_ob gt.stamp('train: fill replay pool') # Train. if self._pool.size >= self._min_pool_size: self._do_training(itr) itr += 1 gt.stamp('train: updates') # Evaluate. self._evaluate(epoch) gt.stamp("test") # Log. params = self.get_epoch_snapshot(epoch) logger.save_itr_params(epoch, params) times_itrs = gt.get_times().stamps.itrs train_time = np.sum([ times_itrs[key][-1] for key in times_itrs.keys() if 'train: ' in key ]) eval_time = times_itrs["test"][-1] total_time = gt.get_times().total logger.record_tabular("time: train", train_time) logger.record_tabular("time: eval", eval_time) logger.record_tabular("time: total", total_time) logger.record_tabular("scale_reward", self._scale_reward) logger.record_tabular("epochs", epoch) logger.record_tabular("steps: all", itr) logger.dump_tabular(with_prefix=False) logger.pop_prefix() gt.stamp("logging") print( gt.report( include_itrs=False, format_options={'itr_name_width': 30}, )) env.terminate()
def experiment(variant): cuda = True ingpu = False R = 84 E = 100 N = 100 if ingpu: from mujoco_torch.core.bridge import MjCudaRender renderer = MjCudaRender(84, 84, E) envs = [] for e in range(E): env = HalfCheetahEnv() envs.append(env) c = Convnet(6, output_activation=torch.tanh, input_channels=3) if cuda: c.cuda() def step(stamp=True): for e in range(E): env = envs[e] env.step(np.random.rand(6)) gt.stamp('step') if stamp else 0 if ingpu: sims = [env.sim for env in envs] env = envs[e] tensor, img = renderer.get_batch_cuda_tensor(sims, False) tensor = Variable(tensor).float() gt.stamp('render') if stamp else 0 else: imgs = [] for e in range(E): env = envs[e] img = env.sim.render(R, R, device_id=1) imgs.append(img) gt.stamp('render') if stamp else 0 imgs = np.array(imgs) tensor = np_to_var(imgs) if cuda: tensor = tensor.cuda() torch.cuda.synchronize() gt.stamp('transfer') if stamp else 0 u = get_numpy(c.forward(tensor).cpu()) torch.cuda.synchronize() gt.stamp('forward') if stamp else 0 # cv2.imshow("img", img) # cv2.waitKey(1) gt.stamp("start") for i in range(10): step(False) gt.stamp("warmstart") for i in gt.timed_for(range(N)): step() gt.stamp("end") print(gt.report(include_itrs=False))
def run_train(self, loaders, info, optimizer=None): ''' training loop ''' # prepare dictionaries loaders_train = dict(filter(lambda x: 'train' in x[0], loaders.items())) assert len(set([len(loader) for loader in loaders_train.values()])) == 1 loaders_valid = dict(filter(lambda x: 'train' not in x[0], loaders.items())) vocabs_in = {'{};{}'.format( loader.dataset.name, loader.dataset.ann_type): loader.dataset.vocab_in for loader in loaders.values()} epoch_length = len(next(iter(loaders_train.values()))) # initialize summary writer for tensorboardX self.summary_writer = SummaryWriter(log_dir=self.args.dout) # dump config with open(os.path.join(self.args.dout, 'config.json'), 'wt') as f: json.dump(vars(self.args), f, indent=2) # optimizer optimizer, schedulers = model_util.create_optimizer_and_schedulers( info['progress'], self.args, self.parameters(), optimizer) # make sure that all train loaders have the same length assert len(set([len(loader) for loader in loaders_train.values()])) == 1 model_util.save_log( self.args.dout, progress=info['progress'], total=self.args.epochs, stage='train', best_loss=info['best_loss'], iters=info['iters']) # display dout print("Saving to: %s" % self.args.dout) for epoch in range(info['progress'], self.args.epochs): print('Epoch {}/{}'.format(epoch, self.args.epochs)) self.train() train_iterators = { key: iter(loader) for key, loader in loaders_train.items()} metrics = {key: collections.defaultdict(list) for key in loaders_train} gt.reset() for _ in tqdm(range(epoch_length), desc='train'): # sample batches batches = data_util.sample_batches( train_iterators, self.args.device, self.pad, self.args) gt.stamp('data fetching', unique=False) # do the forward passes model_outs, losses_train = {}, {} for batch_name, (traj_data, input_dict, gt_dict) in batches.items(): model_outs[batch_name] = self.model.forward( vocabs_in[batch_name.split(':')[-1]], action=gt_dict['action'], **input_dict) info['iters']['train'] += ( len(traj_data) if ':' not in batch_name else 0) gt.stamp('forward pass', unique=False) # compute losses losses_train = self.model.compute_loss( model_outs, {key: gt_dict for key, (_, _, gt_dict) in batches.items()}) # do the gradient step optimizer.zero_grad() sum_loss = sum( [sum(loss.values()) for name, loss in losses_train.items()]) sum_loss.backward() optimizer.step() gt.stamp('optimizer', unique=False) # compute metrics for dataset_name in losses_train.keys(): self.model.compute_metrics( model_outs[dataset_name], batches[dataset_name][2], metrics['train:' + dataset_name]) for key, value in losses_train[dataset_name].items(): metrics['train:' + dataset_name]['loss/' + key].append( value.item()) metrics['train:' + dataset_name]['loss/total'].append( sum_loss.detach().cpu().item()) gt.stamp('metrics', unique=False) if self.args.profile: print(gt.report(include_itrs=False, include_stats=False)) # compute metrics for train print('Computing train and validation metrics...') metrics = {data: {k: sum(v) / len(v) for k, v in metr.items()} for data, metr in metrics.items()} # compute metrics for valid_seen for loader_id, loader in loaders_valid.items(): with torch.no_grad(): metrics[loader_id] = self.run_validation( loader, vocabs_in[loader_id.split(':')[-1]], loader_id, info['iters']) stats = {'epoch': epoch, 'general': { 'learning_rate': optimizer.param_groups[0]['lr']}, **metrics} # save the checkpoint print('Saving models...') model_util.save_model( self, 'model_{:02d}.pth'.format(epoch), stats, optimizer=optimizer) model_util.save_model(self, 'latest.pth', stats, symlink=True) # write averaged stats for loader_id in stats.keys(): if isinstance(stats[loader_id], dict): for stat_key, stat_value in stats[loader_id].items(): # for comparison with old epxs, maybe remove later summary_key = '{}/{}'.format( loader_id.replace(':', '/').replace( 'lmdb/', '').replace(';lang', '').replace(';', '_'), stat_key.replace(':', '/').replace('lmdb/', '')) self.summary_writer.add_scalar( summary_key, stat_value, info['iters']['train']) # dump the training info model_util.save_log( self.args.dout, progress=epoch+1, total=self.args.epochs, stage='train', best_loss=info['best_loss'], iters=info['iters']) model_util.adjust_lr(optimizer, self.args, epoch, schedulers) print('{} epochs are completed, all the models were saved to: {}'.format( self.args.epochs, self.args.dout))
def train_epoch(epoch): print('\nTRAINING : Epoch ' + str(epoch)) model.train() losses = [] logs = [] sample_avgs = [] update_avgs = [] last_time = time.time() metrics = Metrics(tok2i, i2tok, field=TRG) trajectory_sampler = buffer.TrajectorySampler(trainloader) n_updates = 0 oracle_samples_only = args.rollin_beta == 1.0 while n_updates < updates_per_epoch: gt.reset() if oracle_samples_only: start = time.time() trajectory = trajectory_sampler.get_oracle_trajectory(model, Oracle, oracle_flags=oracle_flags) sample_time = (time.time() - start) start = time.time() loss = trajectory_sampler.get_loss(model, trajectory, loss_flags) update_time = (time.time() - start) else: start = time.time() loss = trajectory_sampler.get_mixed_trajectory_loss(model, Oracle, oracle_flags=oracle_flags, beta=args.rollin_beta, loss_flags=loss_flags) sample_time = 0 update_time = (time.time() - start) optimizer.zero_grad() loss.backward() clip_grad_norm_(model.parameters(), args.max_norm) losses.append(loss.item()) optimizer.step() n_updates += 1 sample_avgs.append(sample_time) update_avgs.append(update_time) gt.stamp("buffer updates") if n_updates % 20 == 0: print("%d|%d\t%.3f\tSample: %.3fs\tUpdate: %.3fs" % (epoch, n_updates, round(np.mean(losses), 3), np.mean(sample_avgs), np.mean(update_avgs))) log_tensorboard({'sample_avgs': np.mean(sample_avgs), 'update_avgs': np.mean(update_avgs)}, step=args.logstep) sample_avgs = [] update_avgs = [] # -- Report metrics every `print_every` batches. if n_updates % args.print_every == 0: gt.stamp("report") # Training report computed over the last `print_every` batches. ms = metrics.report('train') ms['train/loss'] = round(np.mean(losses), 2) logs.append('{0} ; loss {1} ; sentence/s {2} ; {3} train {4} '.format( epoch, round(np.mean(losses), 2), int(len(losses) * args.batch_size / (time.time() - last_time)), args.eval_metric, ms.get('train/%s' % args.eval_metric, 0.0), )) args.logstep += 1 last_time = time.time() losses = [] metrics.reset() # -- Validation report with a single batch. metrics.reset() model.eval() batch = next(iter(validloader)) scores, samples = predict_batch(batch) model.train() metrics.update(scores, samples, (batch.trg[0], None)) vms = metrics.report('valid_batch') logs[-1] = logs[-1] + metrics.log(vms, 'valid_batch', ['bleu', 'avg_span', 'f1', 'em', 'depth_score']) metrics.reset() print_samples(samples, (batch.trg[0], None), n=len(batch)) gt.stamp("validation_batch") log_tensorboard(ms, step=args.logstep) log_tensorboard(vms, step=args.logstep) print(logs[-1]) print(gt.report(include_itrs=False, format_options={'itr_name_width': 30})) # -- Checkpointing if n_updates % args.save_every == 0: print('saving checkpoint at epoch {0} batch {1}'.format(epoch, i)) print(os.path.join(args.log_directory, args.expr_name + '.checkpoint')) torch.save({ 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'optimizer_param': args.optimizer, 'loss': loss.item() }, os.path.join(args.log_directory, args.expr_name + '.checkpoint')) model_config.longest_label = model.longest_label with open(os.path.join(args.log_directory, 'model_config.pkl'), 'wb') as f: pickle.dump(model_config, f) print('end : epoch {0} '.format(epoch)) log_tensorboard({'lr': optimizer.param_groups[0]['lr']}, step=args.logstep)
def experiment(variant): comm = MPI.COMM_WORLD rank = comm.Get_rank() n_proc = comm.Get_size() root = 0 gpus = GPUtil.getGPUs() n_gpu = len(gpus) torch.distributed.init_process_group(backend='mpi', world_size=n_proc) E = 20 R = 84 U = 6 cuda = True envs = [] for e in range(rank, E, n_proc): env = HalfCheetahEnv() envs.append(env) sendcounts = np.array(comm.gather(len(envs), root)) i_sendcounts = None u_sendcounts = None if rank == root: i_sendcounts = sendcounts * 3 * R * R u_sendcounts = sendcounts * U c = Convnet(6, output_activation=torch.tanh, input_channels=3) c = torch.nn.parallel.DistributedDataParallel(c) if cuda: c.cuda() # viewer = mujoco_py.MjRenderContextOffscreen(env.sim, device_id=1) # env.sim.add_render_context(viewer) def step(i, stamp=True): imgs = [] if i % 100 == 0: for e in envs: e.reset() for e in envs: img = e.sim.render(R, R, device_id=rank % n_gpu).transpose() imgs.append(img) comm.Barrier() if rank == 0: gt.stamp('render') if stamp else 0 imgs = np.array(imgs) r_imgs = None if rank == 0: r_imgs = np.empty((E, 3, R, R), dtype='uint8') comm.Gatherv(sendbuf=imgs, recvbuf=(r_imgs, i_sendcounts), root=root) if rank == 0: gt.stamp('comm1') if stamp else 0 u = None if rank == 0: torch_img = np_to_var(r_imgs) if cuda: torch_img = torch_img.cuda() torch.cuda.synchronize() gt.stamp('transfer') if stamp else 0 u = get_numpy(c.forward(torch_img).cpu()) torch.cuda.synchronize() gt.stamp('forward') if stamp else 0 r_u = np.empty((len(envs), U), dtype='float32') comm.Scatterv(sendbuf=(u, u_sendcounts), recvbuf=r_u, root=root) if rank == 0: gt.stamp('comm2') if stamp else 0 for i, e in enumerate(envs): e.step(r_u[i, :]) comm.Barrier() if rank == 0: gt.stamp('step') if stamp else 0 for i in range(10): step(i, False) if rank == 0: gt.stamp('start') for i in gt.timed_for(range(100)): step(i) if rank == 0: gt.stamp('end') print( gt.report(include_itrs=False, format_options=dict(itr_num_width=10)))
envs[e].reset() for e in range(E): img = envs[e].sim.render(R, R, device_id=1).transpose() imgs.append(img) gt.stamp('render') if stamp else 0 imgs = np.array(imgs) torch_img = np_to_var(imgs) if cuda: torch_img = torch_img.cuda() torch.cuda.synchronize() gt.stamp('transfer') if stamp else 0 u = get_numpy(c.forward(torch_img).cpu()) torch.cuda.synchronize() gt.stamp('forward') if stamp else 0 for e in range(E): env.step(u[e, :]) gt.stamp('step') if stamp else 0 for i in range(10): step(False) gt.stamp('start') for i in gt.timed_for(range(100)): step() gt.stamp('end') print(gt.report(include_itrs=False, format_options=dict(itr_num_width=10)))
def train_epoch(epoch): print('\nTRAINING : Epoch ' + str(epoch)) model.train() losses = [] logs = [] last_time = time.time() metrics = Metrics(tok2i, i2tok) for i, data in enumerate(trainloader, 0): # -- Actual Training gt.reset() xs, annots = data xs = xs.to(args.device) gt.stamp("load_data") oracle = Oracle(xs, model.n_classes, tok2i, i2tok, **oracle_flags) gt.stamp("create_oracle") max_steps = 2*xs.ne(tok2i['<p>']).sum(1).max()+1 scores, samples, p_oracle = model.forward(num_samples=args.batch_size, oracle=oracle, max_steps=max_steps, return_p_oracle=True) gt.stamp("forward") loss = loss_fn(scores, samples, p_oracle, tok2i['<end>'], **loss_flags) gt.stamp("loss") optimizer.zero_grad() loss.backward() clip_grad_norm_(model.parameters(), args.max_norm) optimizer.step() gt.stamp("backward") losses.append(loss.item()) # -- Report metrics every `print_every` batches. if i % args.print_every == 0: # Training report; loss averaged over the last `print_every` batches. metrics.update(scores, samples, data) gt.stamp("metrics.update") ms = metrics.report('train') ms['train/loss'] = round(np.mean(losses), 2) logs.append('{0} ; loss {1} ; sentence/s {2} ; f1 train {3} '.format( i+1, round(np.mean(losses), 2), int(len(losses) * args.batch_size / (time.time() - last_time)), 0, )) args.logstep += 1 last_time = time.time() losses = [] metrics.reset() scores, samples = predict_batch(data) print_samples(samples, data) gt.stamp("validation_batch") log_tensorboard(ms, step=args.logstep) print(logs[-1]) print(gt.report(include_itrs=False, format_options={'itr_name_width': 30})) # -- Checkpointing if i % args.save_every == 0: print('saving checkpoint at epoch {0} batch {1}'.format(epoch, i)) torch.save(model.state_dict(), os.path.join(args.log_directory, args.expr_name + '.checkpoint')) model_config['longest_label'] = model.longest_label with open(os.path.join(args.log_directory, 'model_config.json'), 'w') as f: json.dump(model_config, f) print('end : epoch {0} '.format(epoch)) log_tensorboard({'lr': optimizer.param_groups[0]['lr']}, step=args.logstep)
import gtimer as gt import test1_b import time time.sleep(0.1) gt.stamp('first') time.sleep(0.2) gt.stamp('second') loop = gt.timed_loop('loop') for i in [1, 2, 3]: next(loop) test1_b.func() test1_b.func() time.sleep(0.1) gt.stamp('loop1') loop.exit() time.sleep(0.1) gt.stamp('third') x = gt.save_pkl('test.pkl') y = gt.load_pkl('test.pkl') print(gt.report(y))