def gen_best_23_error(): data_loader = DataLoader() Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name checkpoint_path = os.path.join("./experiment", Checkpoint.CHECKPOINT_DIR_NAME, 'best') checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model if args.cuda_use: seq2seq = seq2seq.cuda() seq2seq.eval() emb_model = seq2seq.encoder.embedding emb_np = emb_model.weight.cpu().data.numpy() np.save("./data/rl_train_data/emb.npy", emb_np) evaluator = Evaluator(vocab_dict = data_loader.vocab_dict, vocab_list = data_loader.vocab_list, decode_classes_dict = data_loader.decode_classes_dict, decode_classes_list = data_loader.decode_classes_list, loss = NLLLoss(), cuda_use = args.cuda_use) evaluator.gen_rl_data(model = seq2seq, data_loader = data_loader, data_list = data_loader.math23k_train_list, template_flag = False, batch_size = 16, evaluate_type = 0, use_rule = False, mode = args.mode, filename = args.load_name)
class TestEvaluator(TestCase): def setUp(self): self.info = {0: {'amount_correct': 400, 'amount_total': 600}, 1: {'amount_correct': 100, 'amount_total': 1000}} self.evaluator = Evaluator(self.info) def test_inits_correctly(self): self.assertEqual(self.info, self.evaluator.class_info) def test_update_accuracy(self): self.evaluator.update_accuracy(0, 200, 800) self.assertEqual(self.evaluator.class_info[0].amount_correct, 200) self.assertEqual(self.evaluator.class_info[0].amount_total, 800) def test_percent_correct(self): class_info = self.evaluator.class_info[1] self.assertEqual(class_info.amount_correct / class_info.amount_total, .1) def test_total_percent_correct(self): self.assertEqual(self.evaluator.total_percent_correct(), 500/1600)
def evaluate_fn(agent_dir, output_dir, seeds, port): agent = agent_dir.split('/')[-1] if not check_dir(agent_dir): logging.error('Evaluation: %s does not exist!' % agent) return # load config file for env config_dir = find_file(agent_dir + '/data/') if not config_dir: return config = configparser.ConfigParser() config.read(config_dir) # init env env, greedy_policy = init_env(config['ENV_CONFIG'], port=port, naive_policy=True) env.init_test_seeds(seeds) # load model for agent if agent != 'greedy': # init centralized or multi agent model = init_agent(env, config['MODEL_CONFIG'], 0, 0) if model is None: return if not model.load(agent_dir + '/model/'): return else: model = greedy_policy # collect evaluation data evaluator = Evaluator(env, model, output_dir) evaluator.run()
def test(net, test_loader, epoch): net.eval() test_step = len(test_loader) print('\nEvaluating...') with torch.no_grad(): evaluator = Evaluator() for i, sample in enumerate(test_loader): for key in sample: sample[key] = sample[key].cuda() output = net(sample) evaluator.add(output, sample) if (i + 1) % 100 == 0: print('Val Step [{}/{}]'.format(i + 1, test_step)) results = evaluator.evaluate() print('Epoch {}/{}'.format(epoch + 1, args.epoch)) print( '| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |' ) print( '| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |' .format(results['lm_dist'][0], results['lm_dist'][1], results['lm_dist'][2], results['lm_dist'][3], results['lm_dist'][4], results['lm_dist'][5], results['lm_dist'][6], results['lm_dist'][7], results['lm_dist_all']))
def test(args, test_dataset="test"): seq2seq, data_loader = create_model(args) resume_checkpoint = Checkpoint.load(model_only=True) seq2seq.load_state_dict(resume_checkpoint.model) if args.use_cuda: seq2seq = seq2seq.cuda() evaluator = Evaluator( class_dict=data_loader.class_dict, class_list=data_loader.class_list, use_cuda=args.use_cuda ) if test_dataset == "test": test_dataset = data_loader.test_list elif test_dataset == "train": test_dataset = data_loader.train_list seq2seq.eval() with torch.no_grad(): test_temp_acc, test_ans_acc = evaluator.evaluate( model=seq2seq, data_loader=data_loader, data_list=test_dataset, template_flag=True, template_len=False, batch_size=1, beam_width=args.beam, test_log=args.test_log, print_probability=True ) logging.info(f"temp_acc: {test_temp_acc}, ans_acc: {test_ans_acc}") return
def step_three(): data_loader = DataLoader(args) Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name checkpoint_path = os.path.join("./experiment", Checkpoint.CHECKPOINT_DIR_NAME, "best") checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model if args.cuda_use: seq2seq = seq2seq.cuda() seq2seq.eval() evaluator = Evaluator(vocab_dict=data_loader.vocab_dict, vocab_list=data_loader.vocab_list, decode_classes_dict=data_loader.decode_classes_dict, decode_classes_list=data_loader.decode_classes_list, loss=NLLLoss(), cuda_use=args.cuda_use) test_temp_acc, test_ans_acc = evaluator.evaluate( model=seq2seq, data_loader=data_loader, data_list=data_loader.math57k_data_list, template_flag=False, batch_size=64, evaluate_type=0, use_rule=True, mode=args.mode) print(test_temp_acc, test_ans_acc)
def step_one_test(): data_loader = DataLoader(args) #Checkpoint.CHECKPOINT_DIR_NAME = "0120_0030" Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name checkpoint_path = os.path.join("./experiment", Checkpoint.CHECKPOINT_DIR_NAME, "best") checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model if args.cuda_use: seq2seq = seq2seq.cuda() seq2seq.eval() evaluator = Evaluator(vocab_dict=data_loader.vocab_dict, vocab_list=data_loader.vocab_list, decode_classes_dict=data_loader.decode_classes_dict, decode_classes_list=data_loader.decode_classes_list, loss=NLLLoss(), cuda_use=args.cuda_use) name = args.run_flag if name == 'test_23k': test_temp_acc, test_ans_acc = evaluator.evaluate( model=seq2seq, data_loader=data_loader, data_list=data_loader.math23k_test_list, template_flag=True, batch_size=64, evaluate_type=0, use_rule=False, mode=args.mode, post_flag=args.post_flag, name_save=name) print(test_temp_acc, test_ans_acc)
def evaluate_fn(agent_dir, output_dir, seeds, port, demo, policy_type): agent = agent_dir.split('/')[-1] if not check_dir(agent_dir): logging.error('Evaluation: %s does not exist!' % agent) return # load config file for env config_dir = find_file(agent_dir + '/data/') if not config_dir: return config = configparser.ConfigParser() config.read(config_dir) # init env env, greedy_policy = init_env(config['ENV_CONFIG'], port=port, naive_policy=True) logging.info( 'Evaluation: s dim: %d, a dim %d, s dim ls: %r, a dim ls: %r' % (env.n_s, env.n_a, env.n_s_ls, env.n_a_ls)) env.init_test_seeds(seeds) # load model for agent if agent != 'greedy': # init centralized or multi agent if agent == 'a2c': model = A2C(env.n_s, env.n_a, 0, config['MODEL_CONFIG']) elif agent == 'ia2c': model = IA2C(env.n_s_ls, env.n_a_ls, env.n_w_ls, 0, config['MODEL_CONFIG']) elif agent == 'ma2c': model = MA2C(env.n_s_ls, env.n_a_ls, env.n_w_ls, env.n_f_ls, 0, config['MODEL_CONFIG']) elif agent == 'iqld': model = IQL(env.n_s_ls, env.n_a_ls, env.n_w_ls, 0, config['MODEL_CONFIG'], seed=0, model_type='dqn') else: model = IQL(env.n_s_ls, env.n_a_ls, env.n_w_ls, 0, config['MODEL_CONFIG'], seed=0, model_type='lr') if not model.load(agent_dir + '/model/'): return else: model = greedy_policy env.agent = agent # collect evaluation data evaluator = Evaluator(env, model, output_dir, demo=demo, policy_type=policy_type) evaluator.run()
def gen_math57k_error(): data_loader = DataLoader() Checkpoint.CHECKPOINT_DIR_NAME = args.checkpoint_dir_name checkpoint_path = os.path.join("./experiment", Checkpoint.CHECKPOINT_DIR_NAME, args.load_name) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model if args.cuda_use: seq2seq = seq2seq.cuda() seq2seq.eval() evaluator = Evaluator(vocab_dict = data_loader.vocab_dict, vocab_list = data_loader.vocab_list, decode_classes_dict = data_loader.decode_classes_dict, decode_classes_list = data_loader.decode_classes_list, loss = NLLLoss(), cuda_use = args.cuda_use) evaluator.gen_rl_data(model = seq2seq, data_loader = data_loader, data_list = data_loader.math57k_data_list, template_flag = False, batch_size = 16, evaluate_type = 0, use_rule = True, mode = args.mode, filename = args.load_name)
def evaluate_fn(agent_dir, output_dir, seeds, port, demo): agent = agent_dir.split('/')[-1] if not check_dir(agent_dir): logging.error('Evaluation: %s does not exist!' % agent) return # load config file config_dir = find_file(agent_dir + '/data/') if not config_dir: return config = configparser.ConfigParser() config.read(config_dir) # init env env = init_env(config['ENV_CONFIG'], port=port) env.init_test_seeds(seeds) # load model for agent model = init_agent(env, config['MODEL_CONFIG'], 0, 0) if model is None: return model_dir = agent_dir + '/model/' if not model.load(model_dir): return # collect evaluation data evaluator = Evaluator(env, model, output_dir, gui=demo) evaluator.run()
def setUp(self): self.info = {0: {'amount_correct': 400, 'amount_total': 600}, 1: {'amount_correct': 100, 'amount_total': 1000}} self.evaluator = Evaluator(self.info)
def run(self): # checkpoint self.model = self.model.eval() self.dataset, test_gallery = get_initial_test( self.config, test=True) # return dataset instance print("data set len is :", len(self.dataset)) data_gallery, vID_gallery, label_gallery = test_gallery[ 0], test_gallery[1], test_gallery[2] print("sample leve ----------->", self.config.test.sampler) # dataloader define self.data_loader = DataLoader(dataset=self.dataset, batch_size=1, sampler=SequentialSampler(self.dataset), collate_fn=self.collate_fn, num_workers=self.num_workers) len_gallery = len(label_gallery) feature_gallery = self.extract_gallery_feature(data_gallery, len_gallery) probe_feature = list() probe_vID = list() for seq, vID, label, _ in tqdm(self.data_loader): seq = torch.from_numpy(seq).float().cuda() # print(seq.size()) fc, out = self.model(seq) n, num_bin = fc.size() feat = fc.view(n, -1).data.cpu().numpy() for ii in range(n): feat[ii] = feat[ii] / np.linalg.norm(feat[ii]) probe_feature.append(feat) probe_vID += vID test_gallery = feature_gallery, vID_gallery, label_gallery feature_probe = np.concatenate(probe_feature, 0) test_probe = feature_probe, probe_vID self.save_npy(feature_gallery, "feature_gallery.npy") self.save_npy(vID_gallery, "vID_gallery.npy") self.save_npy(label_gallery, "label_gallery.npy") self.save_npy(feature_probe, "feature_probe.npy") self.save_npy(probe_vID, "probe_vID.npy") evaluation = Evaluator(test_gallery, test_probe, self.config) evaluation.run()
def train(config, device, train_loader, epoch): """ train an epoch Args: config (EasyDict): configurations for training device (torch.device): the GPU or CPU used for training train_loader (torch.utils.data.DataLoader): a DataLoader instance object for training set epoch (int): current epoch Returns: err (float): error rate """ losses = AverageMeter() evaluator = Evaluator(config.num_classes) model.train() with tqdm(train_loader) as pbar: pbar.set_description('Train Epoch {}'.format(epoch)) for step, (input_, target) in enumerate(train_loader): # move data to device input_ = torch.tensor(input_, device=device, dtype=torch.float32) target = torch.tensor(target, device=device, dtype=torch.long) # forward and compute loss output = model(input_) loss = criterion(output, target) # backward and update params optimizer.zero_grad() loss.backward() optimizer.step() # record loss and show it in the pbar losses.update(loss.item(), input_.size(0)) postfix = OrderedDict({ 'batch_loss': f'{losses.val:6.4f}', 'running_loss': f'{losses.avg:6.4f}' }) pbar.set_postfix(ordered_dict=postfix) pbar.update() # visualization with TensorBoard total_iter = (epoch - 1) * len(train_loader) + step + 1 writer.add_scalar('training_loss', losses.val, total_iter) # update confusion matrix true = target.cpu().numpy() pred = output.max(dim=1)[1].cpu().numpy() evaluator.update_matrix(true, pred) return evaluator.error()
def get_callbacks(output_subdir, training_model, prediction_model, val_generator): training_model_checkpoint = MultiGPUModelCheckpoint( os.path.join(output_subdir, cfg.training_model_cp_filename), training_model, save_best_only=cfg.save_best_only, monitor='loss', mode='min') prediction_model_checkpoint = PredictionModelCheckpoint( os.path.join(output_subdir, cfg.prediction_model_cp_filename), prediction_model, save_best_only=cfg.save_best_only, monitor='loss', mode='min') evaluator = Evaluator(prediction_model, val_generator, cfg.label_len, cfg.characters, cfg.optimizer, period=cfg.val_iter_period) lr_reducer = ReduceLROnPlateau(factor=cfg.lr_reduction_factor, patience=3, verbose=1, min_lr=0.00001) os.makedirs(cfg.tb_log, exist_ok=True) tensorboard = TensorBoard(log_dir=cfg.tb_log) return [ training_model_checkpoint, prediction_model_checkpoint, evaluator, lr_reducer, tensorboard ]
def train(self, epoch, train_loader): """ 使用训练集训练一个epoch Args: epoch: (int) 第几代训练 train_loader: (torch.utils.data.DataLoader) 训练数据加载器 Returns: err: (float) error rate """ losses = AverageMeter() evaluator = Evaluator(self.cfgs.num_classes) self.model.train() with tqdm(train_loader) as pbar: pbar.set_description('Train Epoch {}'.format(epoch)) for step, (input_, target) in enumerate(train_loader): # move data to device input_ = torch.tensor(input_, device=self.device, dtype=torch.float32) target = torch.tensor(target, device=self.device, dtype=torch.long) # forward and compute loss output = self.model(input_) loss = self.criterion(output, target) # backward and update params self.optimizer.zero_grad() loss.backward() self.optimizer.step() # record loss and show it in the pbar losses.update(loss.item(), input_.size(0)) postfix = OrderedDict({'batch_loss': f'{losses.val:6.4f}', 'running_loss': f'{losses.avg:6.4f}'}) pbar.set_postfix(ordered_dict=postfix) pbar.update() # visualization with TensorBoard total_iter = (epoch - 1) * len(train_loader) + step + 1 self.writer.add_scalar('training_loss', losses.val, total_iter) # update confusion matrix true = target.cpu().numpy() pred = output.max(dim=1)[1].cpu().numpy() evaluator.update_matrix(true, pred) return evaluator.error()
def __init__(self, argv, preprocessor, model_api): self.argv = argv self.emb_loader = EmbeddingLoader(argv) self.evaluator = Evaluator(argv) self.loader = CoNLL09Loader(argv) self.saver = CoNLL09Saver(argv) self.preprocessor = preprocessor(argv) self.model_api = model_api(argv)
def __init__(self, class_dict, class_list, use_cuda): self.test_train_every = 10 self.print_every = 30 self.use_cuda = use_cuda self.pad_idx_in_class = class_dict['PAD_token'] loss_weight = torch.ones(len(class_dict)) loss_weight[self.pad_idx_in_class] = 0 self.loss = nn.NLLLoss(weight=loss_weight, reduction="sum") if use_cuda: self.loss = self.loss.cuda() self.evaluator = Evaluator(class_dict=class_dict, class_list=class_list, use_cuda=use_cuda) return
def train(args): base_dir = args.base_dir dirs = init_dir(base_dir) init_log(dirs['log']) config_dir = args.config_dir copy_file(config_dir, dirs['data']) config = configparser.ConfigParser() config.read(config_dir) in_test, post_test = init_test_flag(args.test_mode) # init env env = init_env(config['ENV_CONFIG']) logging.info('Training: a dim %d, agent dim: %d' % (env.n_a, env.n_agent)) # init step counter total_step = int(config.getfloat('TRAIN_CONFIG', 'total_step')) test_step = int(config.getfloat('TRAIN_CONFIG', 'test_interval')) log_step = int(config.getfloat('TRAIN_CONFIG', 'log_interval')) global_counter = Counter(total_step, test_step, log_step) # init centralized or multi agent seed = config.getint('ENV_CONFIG', 'seed') model = init_agent(env, config['MODEL_CONFIG'], total_step, seed) # disable multi-threading for safe SUMO implementation summary_writer = tf.summary.FileWriter(dirs['log']) trainer = Trainer(env, model, global_counter, summary_writer, in_test, output_path=dirs['data']) trainer.run() # save model final_step = global_counter.cur_step logging.info('Training: save final model at step %d ...' % final_step) model.save(dirs['model'], final_step) # post-training test if post_test: test_dirs = init_dir(base_dir, pathes=['eva_data']) evaluator = Evaluator(env, model, test_dirs['eva_data']) evaluator.run()
def test(net, test_loader, epoch): net.eval() test_step = len(test_loader) print('\nEvaluating...') with torch.no_grad(): evaluator = Evaluator() for i, sample in enumerate(test_loader): for key in sample: if key[0:2] != 'D_': sample[key] = sample[key].cuda() output = net(sample) evaluator.add(output, sample) if (i + 1) % 100 == 0: print('Val Step [{}/{}]'.format(i + 1, test_step)) # j + 1 ? results = evaluator.evaluate() print('Epoch {}/{}'.format(epoch + 1, args.epoch)) print( '| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |' ) print( '| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |' .format(results['lm_dist'][0], results['lm_dist'][1], results['lm_dist'][2], results['lm_dist'][3], results['lm_dist'][4], results['lm_dist'][5], results['lm_dist'][6], results['lm_dist'][7], results['lm_dist_all'])) file = open('results_lr_%.4f_base_%d_de_%d_g_%.2f.txt'% \ (args.learning_rate, \ args.base_epoch, \ args.decay_epoch, \ args.gamma), 'a') file.write('Epoch {}\n'.format(args.base_epoch + epoch + 1)) file.write( '| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |\n' ) file.write( '| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |\n\n' .format(results['lm_dist'][0], results['lm_dist'][1], results['lm_dist'][2], results['lm_dist'][3], results['lm_dist'][4], results['lm_dist'][5], results['lm_dist'][6], results['lm_dist'][7], results['lm_dist_all'])) file.close()
def validate(config, device, val_loader, epoch): """ validate the model Args: config (EasyDict): configurations for training device (torch.device): the GPU or CPU used for training val_loader (torch.utils.data.DataLoader): a DataLoader instance object for validation set epoch (int): current epoch Returns: err: (float) error rate """ losses = AverageMeter() evaluator = Evaluator(config.num_classes) model.eval() with tqdm(val_loader) as pbar: pbar.set_description('Valid Epoch {}'.format(epoch)) for i, (input_, target) in enumerate(val_loader): # move data to GPU input_ = torch.tensor(input_, device=device, dtype=torch.float32) target = torch.tensor(target, device=device, dtype=torch.long) with torch.no_grad(): # compute output and loss output = model(input_) loss = criterion(output, target) # record loss and show it in the pbar losses.update(loss.item(), input_.size(0)) postfix = OrderedDict({ 'batch_loss': f'{losses.val:6.4f}', 'running_loss': f'{losses.avg:6.4f}' }) pbar.set_postfix(ordered_dict=postfix) pbar.update() # update confusion matrix true = target.cpu().numpy() pred = output.max(dim=1)[1].cpu().numpy() evaluator.update_matrix(true, pred) return evaluator.error()
def evaluate(split='valid'): y_true = eval(split + '_data') # Ttensor = eval(split + '_tensor') model.eval() y_score, _, _ = model(train_tensor) y_score.detach_() y_score = y_score.squeeze(0) y_score[train_data.row, train_data.col] = 0 _, rec_items = torch.topk(y_score, args.N, dim=1) # y_pred = torch.gather(Ttensor, 1, rec_items).cpu().numpy() run = sort2query(rec_items[:, 0:args.N]) test = csr2test(y_true.tocsr()) evaluator = Evaluator({'recall', 'map_cut'}) evaluator.evaluate(run, test) result = evaluator.show([ 'recall_5', 'recall_10', 'recall_15', 'recall_20', 'map_cut_5', 'map_cut_10', 'map_cut_15', 'map_cut_20' ]) print(result)
def evaluate_fn(agent_dir, output_dir, seeds, port): agent = agent_dir.split('/')[-1] if not check_dir(agent_dir): logging.error('Evaluation: %s does not exist!' % agent) return # load config file for env config_dir = find_file(agent_dir) if not config_dir: return config = configparser.ConfigParser() config.read(config_dir) # init env env, greedy_policy = init_env(config['ENV_CONFIG'], port=port, naive_policy=True) env.init_test_seeds(seeds) # load model for agent if agent != 'greedy': # init centralized or multi agent if env.agent == 'ia2c': model = IA2C(env.n_s_ls, env.n_a, env.neighbor_mask, env.distance_mask, env.coop_gamma, 0, config['MODEL_CONFIG']) elif env.agent == 'ia2c_fp': model = IA2C_FP(env.n_s_ls, env.n_a, env.neighbor_mask, env.distance_mask, env.coop_gamma, 0, config['MODEL_CONFIG']) elif env.agent == 'ma2c_nc': model = MA2C_NC(env.n_s, env.n_a, env.neighbor_mask, env.distance_mask, env.coop_gamma, 0, config['MODEL_CONFIG']) else: return if not model.load(agent_dir + '/'): return else: model = greedy_policy env.agent = agent # collect evaluation data evaluator = Evaluator(env, model, output_dir) evaluator.run()
def test(segmentation_module, args=None): label_num_ = args.num_class segmentation_module.eval() evaluator = Evaluator(label_num_) print('validation') with open(os.path.join(args.dataroot, 'val.txt'), 'r') as f: lines = f.readlines() videolists = [line[:-1] for line in lines] for video in videolists: test_dataset = TestDataset_clip(args.dataroot, video, args, is_train=True) loader = torch.utils.data.DataLoader(test_dataset, batch_size=args.batchsize, shuffle=False, num_workers=args.workers, drop_last=False) for i, data in enumerate(loader): # process data print('[{}]/[{}]'.format(i, len(loader))) imgs, gts, clip_imgs, _, _ = data imgs = imgs.cuda(args.start_gpu) gts = gts.cuda(args.start_gpu) clip_imgs = [img.cuda(args.start_gpu) for img in clip_imgs] batch_data = {} batch_data['img_data'] = imgs batch_data['seg_label'] = gts batch_data['clipimgs_data'] = clip_imgs segSize = (imgs.size(2), imgs.size(3)) with torch.no_grad(): scores = segmentation_module(batch_data, segSize=segSize) pred = torch.argmax(scores, dim=1) pred = pred.data.cpu().numpy() target = gts.squeeze(1).cpu().numpy() # Add batch sample into evaluator evaluator.add_batch(target, pred) Acc = evaluator.Pixel_Accuracy() Acc_class = evaluator.Pixel_Accuracy_Class() mIoU = evaluator.Mean_Intersection_over_Union() FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union() print('Validation:') print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format( Acc, Acc_class, mIoU, FWIoU))
def init_trainer(self, model, epochs, lr, eval_steps=1, metric='acc'): self.epochs = epochs self.lr = lr self.eval_steps = eval_steps self.metric = metric self.model = model self.optimizer = torch.optim.Adam(model.parameters(), lr=self.lr, weight_decay=5e-4) self.evaluator = Evaluator(metric=self.metric) self.best_model_parameters = None
def train(self, model, data_loader, batch_size, n_epoch, template_flag, \ resume=False, optimizer=None, mode=0, teacher_forcing_ratio=0, post_flag=False): self.evaluator = Evaluator( vocab_dict=self.vocab_dict, vocab_list=self.vocab_list, decode_classes_dict=self.decode_classes_dict, decode_classes_list=self.decode_classes_list, loss=NLLLoss(), cuda_use=self.cuda_use) if resume: checkpoint_path = Checkpoint.get_certain_checkpoint( "./experiment", "best") resume_checkpoint = Checkpoint.load(checkpoint_path) model = resume_checkpoint.model self.optimizer = resume_checkpoint.optimizer resume_optim = self.optimizer.optimizer defaults = resume_optim.param_groups[0] defaults.pop('params', None) self.optimizer.optimizer = resume_optim.__class__( model.parameters(), **defaults) start_epoch = resume_checkpoint.epoch start_step = resume_checkpoint.step self.train_acc_list = resume_checkpoint.train_acc_list self.test_acc_list = resume_checkpoint.test_acc_list self.loss_list = resume_checkpoint.loss_list else: start_epoch = 1 start_step = 0 self.train_acc_list = [] self.test_acc_list = [] self.loss_list = [] model_opt = NoamOpt( 512, 1, 2000, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9)) if optimizer is None: optimizer = Optimizer(optim.Adam(model.parameters()), max_grad_norm=0) self.optimizer = model_opt self._train_epoches(data_loader=data_loader, model=model, batch_size=batch_size, start_epoch=start_epoch, start_step=start_step, n_epoch=n_epoch, mode=mode, template_flag=template_flag, teacher_forcing_ratio=teacher_forcing_ratio, post_flag=post_flag)
def test(net, test_loader, epoch): net.eval() test_step = len(test_loader) print('\nEvaluating...') with torch.no_grad(): evaluator = Evaluator() for i, (sample, img) in enumerate(test_loader): #import matplotlib.pyplot as plt #print(len(sample['image_original'])) #plt.imshow(sample['image_original'].numpy().swapaxes(0, 2)) #plt.show() #image_name = sample['image_name'] #print(image_name) for key in sample: if key != 'image_name': sample[key] = sample[key].cuda() output = net(sample) #print(type(output)) pos_map = output['lm_pos_map'] #print(pos_map.size()) #resh = pos_map.reshape(50, 8, -1) #print(resh.size()) evaluator.add(output, sample, img) if (i + 1) % 100 == 0: print('Val Step [{}/{}]'.format(i + 1, test_step)) results = evaluator.evaluate() print('Epoch {}/{}'.format(epoch + 1, args.epoch)) print( '| L.Collar | R.Collar | L.Sleeve | R.Sleeve | L.Waist | R.Waist | L.Hem | R.Hem | ALL |' ) print( '| {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} | {:.5f} |' .format(results['lm_dist'][0], results['lm_dist'][1], results['lm_dist'][2], results['lm_dist'][3], results['lm_dist'][4], results['lm_dist'][5], results['lm_dist'][6], results['lm_dist'][7], results['lm_dist_all']))
def predict(): net = UNet(n_channels=3, n_classes=2) # net = ULeNet(n_channels=3, n_classes=6) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') net.to(device=device) net.load_state_dict(torch.load(model, map_location=device)) evaluator = Evaluator(num_class=2) evaluator.reset() for img in os.listdir(dir_img): image = cv2.imread(os.path.join(dir_img, img)) # mask = cv2.imread(os.path.join(dir_mask, img)) # mask = cv2.cvtColor(mask, cv2.COLOR_RGB2GRAY) mask = Image.open(os.path.join(dir_mask, img)) mask = mask.convert('L') mask = np.array(mask) mask_class = np.unique(mask) for i in range(len(mask_class)): mask[mask == mask_class[i]] = i predict = np.zeros((image.shape[0], image.shape[1])) p_size = 128 for i in range(0, image.shape[0] - p_size, p_size): for j in range(0, image.shape[1] - p_size, p_size): patch = image[i:i + p_size, j:j + p_size, :] patch = Normalization(patch) predict[i:i + p_size, j:j + p_size] = getPredict(net=net, img=patch, device=device) evaluator.add_batch(mask, predict) # mIoU = evaluator.Mean_Intersection_over_Union() predict = Image.fromarray((predict).astype(np.uint8)) predict.save( os.path.join(dir_predict, os.path.splitext(img)[0] + '.tif')) show_confMat(evaluator.conf_mat, [str(c) for c in range(2)], re.split('[/.]', model)[1], dir_confmat)
def eval_net(net, data_loader, device): net.eval() val_batch_num = len(data_loader) eval_loss = 0 e = Evaluator(num_class=8) pixel_acc_avg = 0 mean_iou_avg = 0 fw_iou_avg = 0 with tqdm(total=val_batch_num, desc='Validation round', unit='batch', leave=False) as pbar: for idx, batch_samples in enumerate(data_loader): batch_image, batch_mask = batch_samples["image"], batch_samples[ "mask"] batch_image = batch_image.to(device=device, dtype=torch.float32) mask_true = batch_mask.to(device=device, dtype=torch.long) with torch.no_grad(): mask_pred = net(batch_image) probs = F.softmax(mask_pred, dim=1).squeeze(0) # [8, 256, 256] pre = torch.argmax(probs, dim=1) # [256,256] #???? e.add_batch(mask_true.cpu().data.numpy(), pre.cpu().data.numpy()) pixel_acc = e.Pixel_Accuracy() pixel_acc_avg += pixel_acc mean_iou = e.Mean_Intersection_over_Union() mean_iou_avg += mean_iou fw_iou = e.Frequency_Weighted_Intersection_over_Union() fw_iou_avg += fw_iou eval_loss += F.cross_entropy(mask_pred, mask_true).item() pbar.set_postfix({'eval_loss': eval_loss / (idx + 1)}) pbar.update() e.reset() print("pixel_acc_avg:" + str(pixel_acc_avg / val_batch_num)) print("mean_iou_avg:" + str(mean_iou_avg / val_batch_num)) print("fw_iou_avg:" + str(fw_iou_avg / val_batch_num)) net.train() return eval_loss / val_batch_num, pixel_acc_avg / val_batch_num, mean_iou_avg / val_batch_num, fw_iou_avg / val_batch_num
def fit(self, iteration=1000): """ generate a style-transfered image. The overall process is as follows: 1. initiating the initial canvas 2. setting Evaluator 3. optimizing using L-BFGS method Args: iteration (int): the total iteration number of optimization """ if self.initial_canvas == 'random': generated_img = utils.preproc(np.random.uniform(0, 255, size=self.img_shape)\ .astype(K.floatx())) elif self.initial_canvas == 'content': generated_img = self.content_img.copy() else: # style generated_img = self.style_img.copy() evaluator = Evaluator(self.eval_fn, self.img_shape) self.step = 0 print('Starting optimization with L-BFGS-B') for i in range(1, iteration + 1): if self.verbose: print('Starting iteration %d' % (i)) start_time = time.time() generated_img, min_loss, _ = fmin_l_bfgs_b(evaluator.loss, generated_img.flatten(), fprime=evaluator.grads, callback=self._callback, maxfun=20) generated_img = np.clip(generated_img, -127, 127) end_time = time.time() if self.verbose: print('Total_Loss: %g, Content Loss: %g, Style Loss: %g' % \ (min_loss, evaluator.content_loss, evaluator.style_loss)) print('Iteration %d completed in %d s' % (i, end_time - start_time)) if i == 1 or (self.save_every_n_iters != 0 and i % self.save_every_n_iters == 0): utils.save_image(utils.deproc(generated_img, self.img_shape), self.output_path, 'generated_%d' % (i) + '.jpg')
def test(net, test_loader, epoch): net.eval() test_step = len(test_loader) print('\nEvaluating...') with torch.no_grad(): evaluator = Evaluator() pbar2 = tqdm(test_loader) for i, sample in enumerate(pbar2): iter_start_time = time.time() for key in sample: sample[key] = sample[key].cuda() output = net(sample) evaluator.add(output, sample) t = time.time() - iter_start_time if (i + 1) % 100 == 0: tqdm.write('Val Step [{}/{}], Time:{:.3f}'.format( i + 1, test_step, t)) results = evaluator.evaluate() print('Epoch {}/{}'.format(epoch + 1, args.epoch)) print('lm_dist_all: {:.5f} '.format(results['lm_dist_all']))