def get_accuracy_score(targets, predictions): assert len(targets) == len(predictions) met = [] for i in range(len(targets)): met.append(accuracy(predictions[i], targets[i]).item()) return np.mean(met)
def test_accuracy(): x = torch.rand(8, 10) y = torch.rand(8).long() acc = metrics.accuracy(x, y) assert type(acc) is float assert 0.0 <= acc <= 1.0
def ood_data_experiment(): """Repeat experiment from Ovidia et al. (2019), plotting boxplots with accuracy and ece on corrupted data""" data_dir = "../../dataloaders/data/" model_list = ["distilled", "dirichlet_distill", "mixture_distill", "ensemble_new", "vanilla", "temp_scaling", "dropout_nofirst", "ll_dropout", "svi", "ll_svi"] corruption_list = ["brightness", "contrast", "defocus_blur", "elastic_transform", "fog", "frost", "gaussian_blur", "gaussian_noise", "glass_blur", "impulse_noise", "pixelate", "saturate", "shot_noise", "spatter", "speckle_noise", "zoom_blur"] intensity_list = [0, 1, 2, 3, 4, 5] rep_list = [1, 2, 3, 4, 5] ensemble_inds = np.load("data/ensemble_indices.npy") ensemble_size = 10 acc_list = [] ece_list = [] for intensity in intensity_list: print(intensity) acc = np.zeros((len(corruption_list), len(rep_list), len(model_list))) ece = np.zeros((len(corruption_list), len(rep_list), len(model_list))) for i, corruption in enumerate(corruption_list): print(corruption) for j, model in enumerate(model_list): for k, rep in enumerate(rep_list): if model == "ensemble_new": inds = ensemble_inds[((rep-1) * ensemble_size):(rep * ensemble_size)] else: inds = None data = cifar10_benchmark_model_predictions.Cifar10DataPredictions(model=model, corruption=corruption, intensity=intensity, data_dir=data_dir, rep=rep, ensemble_indices=inds) if model in ["distilled", "ensemble_new", "dirichlet_distill"]: data.set.predictions = np.mean(data.set.predictions, axis=1) acc[i, k-1, j] = metrics.accuracy(torch.tensor(data.set.predictions), torch.tensor(data.set.targets, dtype=torch.long)) ece[i, k-1, j] = metrics.ece(data.set.predictions, data.set.targets) acc_list.append(acc.reshape(-1, acc.shape[-1])) ece_list.append(ece.reshape(-1, ece.shape[-1])) model_list_text = ["Gaussian Distilled", "Dirichlet Distilled", "Mixture Distilled", "Ensemble", "Vanilla", "Temp Scaling", "Dropout", "LL Dropout", "SVI", "LL SVI"] colors = ["#A6CEE3", "#1F78B4", "#B2DF8A", "#33A02C", "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00", "#CAB2D6", "#6A3D9A"] make_boxplot(acc_list, "data/fig/acc_reduced_benchmark_experiments.tikz", model_list=model_list_text, colors=colors) make_boxplot(ece_list, "data/fig/ece_reduced_benchmark_experiments.tikz", label="ECE", model_list=model_list_text, colors=colors, max_y=0.8)
def evaluate(model, data_input, gold_output): predictions = model.predict(data_input, batch_size=config.Params.batch_size, verbose=1) if len(predictions.shape) == 3: predictions_classes = np.argmax(predictions, axis=2) train_batch_f1 = metrics.accuracy_per_sentence(predictions_classes, gold_output) print("Results (per sentence): ", train_batch_f1) train_y_properties_stream = gold_output.reshape(gold_output.shape[0] * gold_output.shape[1]) predictions_classes = predictions_classes.reshape(predictions_classes.shape[0] * predictions_classes.shape[1]) class_mask = train_y_properties_stream != 0 train_y_properties_stream = train_y_properties_stream[class_mask] predictions_classes = predictions_classes[class_mask] else: predictions_classes = np.argmax(predictions, axis=1) train_y_properties_stream = gold_output accuracy = metrics.accuracy(predictions_classes, train_y_properties_stream) micro_scores = metrics.compute_micro_PRF(predictions_classes, train_y_properties_stream, empty_label=keras_models.p0_index) print("Results: Accuracy: ", accuracy) print("Results: Micro-Average F1: ", micro_scores) return predictions_classes, predictions
def evaluate(self, model, device): model.eval() eval_loss, eval_accuracy = 0, 0 nb_eval_steps, nb_eval_examples = 0, 0 for input_ids, input_mask, segment_ids, label_ids in tqdm( self.evaluation_data_loader, desc="Evaluating"): input_ids = input_ids.to(device) input_mask = input_mask.to(device) segment_ids = segment_ids.to(device) label_ids = label_ids.to(device) with torch.no_grad(): tmp_eval_loss = model(input_ids, segment_ids, input_mask, label_ids) logits = model(input_ids, segment_ids, input_mask) logits = logits.detach().cpu().numpy() label_ids = label_ids.to('cpu').numpy() tmp_eval_accuracy = accuracy(logits, label_ids) eval_loss += tmp_eval_loss.mean().item() eval_accuracy += tmp_eval_accuracy nb_eval_examples += input_ids.size(0) nb_eval_steps += 1 eval_loss = eval_loss / nb_eval_steps eval_accuracy = eval_accuracy / nb_eval_examples result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy} with open(self.output_path, "a+") as writer: tqdm.write("***** Eval results *****") writer.write("***** Eval results *****\n") for key in sorted(result.keys()): tqdm.write("{}: {}".format(key, str(result[key]))) writer.write("{}: {}\n".format(key, str(result[key])))
def build_model(): """Скрипт для потсроения модели""" print('reading/preparing train data') raw_data = read_csv_data(train_csv_path) assert all([f in raw_data.columns.tolist() for f in feats ]), 'not all feature columns are present in train data' assert 'y' in raw_data, 'target column is not present in data' assert raw_data.shape[0] > N_NEIGHBORS X = raw_data[feats].values y = raw_data['y'].values assert not (X == X[0]).all(), 'all rows in training data are the same' assert np.unique( y).shape[0] > 1, 'target data must contain more than 1 unique value' X_train, y_train, X_val, y_val = train_test_split(X, y, 0.2) print('training model') mdl = ModelPipeline(steps) mdl.fit(X_train, y_train) y_pred = mdl.predict(X_val) acc = accuracy(y_pred, y_val) print(f'validation accuracy: {acc}') print('saving model') if not os.path.exists(saved_models_path): os.makedirs(saved_models_path) pickle.dump(mdl, open(os.path.join(saved_models_path, 'mdl.p'), 'wb')) print('done')
def job(tuning, params_path, devices, resume, save_interval): global params if tuning: with open(params_path, 'r') as f: params = json.load(f) mode_str = 'tuning' setting = '_'.join(f'{tp}-{params[tp]}' for tp in params['tuning_params']) else: mode_str = 'train' setting = '' # パラメーターを変えるときにseedも変えたい(seed averagingの効果を期待) seed = sum(ord(_) for _ in str(params.values())) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) torch.backends.cudnn.benchmark = False exp_path = ROOT + f'experiments/{params["ex_name"]}/' os.environ['CUDA_VISIBLE_DEVICES'] = devices logger, writer = utils.get_logger( log_dir=exp_path + f'{mode_str}/log/{setting}', tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}') if params['augmentation'] == 'soft': params['scale_limit'] = 0.2 params['brightness_limit'] = 0.1 elif params['augmentation'] == 'middle': params['scale_limit'] = 0.3 params['shear_limit'] = 4 params['brightness_limit'] = 0.1 params['contrast_limit'] = 0.1 else: raise ValueError train_transform, eval_transform = data_utils.build_transforms( scale_limit=params['scale_limit'], shear_limit=params['shear_limit'], brightness_limit=params['brightness_limit'], contrast_limit=params['contrast_limit'], ) data_loaders = data_utils.make_train_loaders( params=params, data_root=ROOT + 'input/' + params['data'], train_transform=train_transform, eval_transform=eval_transform, scale='S', test_size=0, class_topk=params['class_topk'], num_workers=8) model = models.LandmarkNet( n_classes=params['class_topk'], model_name=params['model_name'], pooling=params['pooling'], loss_module=params['loss'], s=params['s'], margin=params['margin'], theta_zero=params['theta_zero'], use_fc=params['use_fc'], fc_dim=params['fc_dim'], ).cuda() optimizer = utils.get_optim(params, model) criterion = nn.CrossEntropyLoss() scheduler = optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=params['epochs'] * len(data_loaders['train']), eta_min=3e-6) start_epoch = 0 if len(devices.split(',')) > 1: model = nn.DataParallel(model) for epoch in range(start_epoch, params['epochs']): logger.info( f'Epoch {epoch}/{params["epochs"]} | lr: {optimizer.param_groups[0]["lr"]}' ) # ============================== train ============================== # model.train(True) losses = utils.AverageMeter() prec1 = utils.AverageMeter() for i, (_, x, y) in tqdm(enumerate(data_loaders['train']), total=len(data_loaders['train']), miniters=None, ncols=55): x = x.to('cuda') y = y.to('cuda') outputs = model(x, y) loss = criterion(outputs, y) optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() acc = metrics.accuracy(outputs, y) losses.update(loss.item(), x.size(0)) prec1.update(acc, x.size(0)) if i % 100 == 99: logger.info( f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}' ) train_loss = losses.avg train_acc = prec1.avg writer.add_scalars('Loss', {'train': train_loss}, epoch) writer.add_scalars('Acc', {'train': train_acc}, epoch) writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch) if (epoch + 1) == params['epochs'] or (epoch + 1) % save_interval == 0: output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth' utils.save_checkpoint(path=output_file_name, model=model, epoch=epoch, optimizer=optimizer, params=params) model = model.module datasets = ('roxford5k', 'rparis6k') results = eval_datasets(model, datasets=datasets, ms=False, tta_gem_p=1.0, logger=logger) if tuning: tuning_result = {} for d in datasets: for key in ['mapE', 'mapM', 'mapH']: mapE, mapM, mapH, mpE, mpM, mpH, kappas = results[d] tuning_result[d + '-' + key] = [eval(key)] utils.write_tuning_result(params, tuning_result, exp_path + 'tuning/results.csv')
total=len(data_loaders['train']), miniters=None, ncols=55): if num_GPU>0: x = x.to('cuda') y = y.to('cuda') outputs = model(x, y) loss = criterion(outputs, y) optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() acc = metrics.accuracy(outputs, y) losses.update(loss.item(), x.size(0)) prec1.update(acc, x.size(0)) if i % 100 == 99: logger.info(f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}') train_loss = losses.avg train_acc = prec1.avg print("[{:5d}] => loss={:.9f}, acc={:.9f}; lr={:.9f}.".format(epoch, train_loss, train_acc)) writer.add_scalars('Loss', {'train': train_loss}, epoch) writer.add_scalars('Acc', {'train': train_acc}, epoch) writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch)
def train_distilled_network_dirichlet( model_dir="models/distilled_model_cifar10_dirichlet"): """Distill ensemble with distribution distillation (Dirichlet) """ args = utils.parse_args() log_file = Path("{}.log".format(datetime.now().strftime('%Y%m%d_%H%M%S'))) utils.setup_logger(log_path=Path.cwd() / args.log_dir / log_file, log_level=args.log_level) data_ind = np.load( "src/experiments/cifar10/training_files/training_data_indices.npy") num_train_points = 40000 train_ind = data_ind[:num_train_points] valid_ind = data_ind[num_train_points:] train_data = cifar10_ensemble_pred.Cifar10Data(ind=train_ind, augmentation=True) valid_data = cifar10_ensemble_pred.Cifar10Data(ind=valid_ind) train_loader = torch.utils.data.DataLoader(train_data.set, batch_size=100, shuffle=True, num_workers=0) valid_loader = torch.utils.data.DataLoader(valid_data.set, batch_size=100, shuffle=True, num_workers=0) test_data = cifar10_ensemble_pred.Cifar10Data(train=False) test_loader = torch.utils.data.DataLoader(test_data.set, batch_size=64, shuffle=True, num_workers=0) ensemble_size = 10 # Note that the ensemble predictions are assumed to have been saved to file (see ensemble_predictions.py), # ensemble_indices.npy contains the order of the ensemble members such that ind[:ensemble_size] are the indices # of the first ensemble, ind[ensemble_size:2*ensemble_size] are the indices of the second ensemble and so on ind = np.load("src/experiments/cifar10/training_files/ensemble_indices.npy" )[((args.rep - 1) * ensemble_size):(args.rep * ensemble_size)] ensemble = ensemble_wrapper.EnsembleWrapper(output_size=10, indices=ind) device = utils.torch_settings(args.seed, args.gpu) distilled_model = cifar_resnet_dirichlet.CifarResnetDirichlet( ensemble, resnet_utils.BasicBlock, [3, 2, 2, 2], device=device, learning_rate=args.lr) loss_metric = metrics.Metric(name="Mean loss", function=distilled_model.calculate_loss) distilled_model.add_metric(loss_metric) distilled_model.train(train_loader, num_epochs=args.num_epochs, validation_loader=valid_loader) distilled_model.eval_mode() predicted_distribution = [] all_labels = [] for batch in test_loader: inputs, labels = batch inputs, labels = inputs[0].to(distilled_model.device), labels.to( distilled_model.device) predicted_distribution.append( distilled_model.predict(inputs).to(distilled_model.device)) all_labels.append(labels.long()) test_acc = metrics.accuracy(torch.cat(predicted_distribution), torch.cat(all_labels)) LOGGER.info("Test accuracy is {}".format(test_acc)) torch.save(distilled_model.state_dict(), model_dir)
def job(tuning, params_path, devices, resume, save_interval): global params if tuning: with open(params_path, 'r') as f: params = json.load(f) mode_str = 'tuning' setting = '_'.join(f'{tp}-{params[tp]}' for tp in params['tuning_params']) else: mode_str = 'train' setting = '' exp_path = ROOT + f'experiments/{params["ex_name"]}/' os.environ['CUDA_VISIBLE_DEVICES'] = devices if resume is None: # C-AIRとABCIで整合性が取れるようにしている。 params[ 'base_ckpt_path'] = f'experiments/v1only/ep4_augmentation-soft_epochs-5_loss-{params["loss"]}.pth' params[ 'clean_path'] = ROOT + f'input/clean/train19_cleaned_verifythresh{params["verifythresh"]}_freqthresh{params["freqthresh"]}.csv' else: params = utils.load_checkpoint(path=resume, params=True)['params'] logger, writer = utils.get_logger( log_dir=exp_path + f'{mode_str}/log/{setting}', tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}') if params['augmentation'] == 'soft': params['scale_limit'] = 0.2 params['brightness_limit'] = 0.1 elif params['augmentation'] == 'middle': params['scale_limit'] = 0.3 params['shear_limit'] = 4 params['brightness_limit'] = 0.1 params['contrast_limit'] = 0.1 else: raise ValueError train_transform, eval_transform = data_utils.build_transforms( scale_limit=params['scale_limit'], shear_limit=params['shear_limit'], brightness_limit=params['brightness_limit'], contrast_limit=params['contrast_limit'], ) data_loaders = data_utils.make_train_loaders( params=params, data_root=ROOT + 'input/' + params['data'], train_transform=train_transform, eval_transform=eval_transform, scale='SS2', test_size=0, class_topk=params['class_topk'], num_workers=8) model = models.LandmarkNet( n_classes=params['class_topk'], model_name=params['model_name'], pooling=params['pooling'], loss_module=params['loss'], s=params['s'], margin=params['margin'], theta_zero=params['theta_zero'], use_fc=params['use_fc'], fc_dim=params['fc_dim'], ).cuda() criterion = nn.CrossEntropyLoss() optimizer = utils.get_optim(params, model) if resume is None: sdict = torch.load(ROOT + params['base_ckpt_path'])['state_dict'] if params['loss'] == 'adacos': del sdict['final.W'] # remove fully-connected layer elif params['loss'] == 'softmax': del sdict['final.weight'], sdict[ 'final.bias'] # remove fully-connected layer else: del sdict['final.weight'] # remove fully-connected layer model.load_state_dict(sdict, strict=False) scheduler = optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=params['epochs'] * len(data_loaders['train']), eta_min=3e-6) start_epoch, end_epoch = (0, params['epochs'] - params['scaleup_epochs']) else: ckpt = utils.load_checkpoint(path=resume, model=model, optimizer=optimizer, epoch=True) model, optimizer, start_epoch = ckpt['model'], ckpt[ 'optimizer'], ckpt['epoch'] + 1 end_epoch = params['epochs'] scheduler = optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=params['epochs'] * len(data_loaders['train']), eta_min=3e-6, last_epoch=start_epoch * len(data_loaders['train'])) setting += 'scaleup_' + resume.split('/')[-1].replace('.pth', '') data_loaders = data_utils.make_verified_train_loaders( params=params, data_root=ROOT + 'input/' + params['data'], train_transform=train_transform, eval_transform=eval_transform, scale='M2', test_size=0, num_workers=8) batch_norm.freeze_bn(model) if len(devices.split(',')) > 1: model = nn.DataParallel(model) for epoch in range(start_epoch, end_epoch): logger.info(f'Epoch {epoch}/{end_epoch}') # ============================== train ============================== # model.train(True) losses = utils.AverageMeter() prec1 = utils.AverageMeter() for i, (_, x, y) in tqdm(enumerate(data_loaders['train']), total=len(data_loaders['train']), miniters=None, ncols=55): x = x.to('cuda') y = y.to('cuda') outputs = model(x, y) loss = criterion(outputs, y) optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() acc = metrics.accuracy(outputs, y) losses.update(loss.item(), x.size(0)) prec1.update(acc, x.size(0)) if i % 100 == 99: logger.info( f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}' ) train_loss = losses.avg train_acc = prec1.avg writer.add_scalars('Loss', {'train': train_loss}, epoch) writer.add_scalars('Acc', {'train': train_acc}, epoch) writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch) if (epoch + 1) == end_epoch or (epoch + 1) % save_interval == 0: output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth' utils.save_checkpoint(path=output_file_name, model=model, epoch=epoch, optimizer=optimizer, params=params) model = model.module datasets = ('oxford5k', 'paris6k', 'roxford5k', 'rparis6k') results = eval_datasets(model, datasets=datasets, ms=True, tta_gem_p=1.0, logger=logger) if tuning: tuning_result = {} for d in datasets: if d in ('oxford5k', 'paris6k'): tuning_result[d] = results[d] else: for key in ['mapE', 'mapM', 'mapH']: mapE, mapM, mapH, mpE, mpM, mpH, kappas = results[d] tuning_result[d + '-' + key] = [eval(key)] utils.write_tuning_result(params, tuning_result, exp_path + 'tuning/results.csv')
def job(tuning, params_path, devices, resume, save_interval): global params if tuning: with open(params_path, 'r') as f: params = json.load(f) mode_str = 'tuning' setting = '_'.join(f'{tp}-{params[tp]}' for tp in params['tuning_params']) else: mode_str = 'train' setting = '' exp_path = ROOT + f'experiments/{params["ex_name"]}/' os.environ['CUDA_VISIBLE_DEVICES'] = devices logger, writer = utils.get_logger( log_dir=exp_path + f'{mode_str}/log/{setting}', tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}') train_transform, eval_transform = build_transforms( scale_range=params['scale_range'], brightness_range=params['brightness_range']) data_loaders = data_utils.make_train_loaders( params=params, data_root=ROOT + 'input/train2018', train_transform=train_transform, eval_transform=eval_transform, class_topk=params['class_topk'], num_workers=8) model = models.LandmarkFishNet( n_classes=params['class_topk'], model_name=params['model_name'], pooling_strings=params['pooling'].split(','), loss_module='arcface', s=30.0, margin=params['margin'], use_fc=params['use_fc'], fc_dim=params['fc_dim'], ).cuda() optimizer = utils.get_optim(params, model) criterion = nn.CrossEntropyLoss() scheduler = optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=params['epochs'] * len(data_loaders['train']), eta_min=1e-6) if len(devices.split(',')) > 1: model = nn.DataParallel(model) if resume is not None: model, optimizer = utils.load_checkpoint(path=resume, model=model, optimizer=optimizer) for epoch in range(params['epochs']): logger.info( f'Epoch {epoch}/{params["epochs"]} | lr: {optimizer.param_groups[0]["lr"]}' ) # ============================== train ============================== # model.train(True) losses = utils.AverageMeter() prec1 = utils.AverageMeter() for i, (_, x, y) in tqdm(enumerate(data_loaders['train']), total=len(data_loaders['train']), miniters=None, ncols=55): x = x.to('cuda') y = y.to('cuda') outputs = model(x, y) loss = criterion(outputs, y) optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() acc = metrics.accuracy(outputs, y) losses.update(loss.item(), x.size(0)) prec1.update(acc, x.size(0)) if i % 100 == 99: logger.info( f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}' ) train_loss = losses.avg train_acc = prec1.avg # ============================== validation ============================== # model.train(False) losses.reset() prec1.reset() for i, (_, x, y) in tqdm(enumerate(data_loaders['val']), total=len(data_loaders['val']), miniters=None, ncols=55): x = x.to('cuda') y = y.to('cuda') with torch.no_grad(): outputs = model(x, y) loss = criterion(outputs, y) acc = metrics.accuracy(outputs, y) losses.update(loss.item(), x.size(0)) prec1.update(acc, x.size(0)) val_loss = losses.avg val_acc = prec1.avg logger.info(f'[Val] Loss: \033[1m{val_loss:.4f}\033[0m | ' f'Acc: \033[1m{val_acc:.4f}\033[0m\n') writer.add_scalars('Loss', {'train': train_loss}, epoch) writer.add_scalars('Acc', {'train': train_acc}, epoch) writer.add_scalars('Loss', {'val': val_loss}, epoch) writer.add_scalars('Acc', {'val': val_acc}, epoch) writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch) if save_interval > 0: if (epoch + 1) == params['epochs'] or (epoch + 1) % save_interval == 0: output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth' utils.save_checkpoint(path=output_file_name, model=model, epoch=epoch, optimizer=optimizer, params=params) if tuning: tuning_result = {} for key in ['train_loss', 'train_acc', 'val_loss', 'val_acc']: tuning_result[key] = [eval(key)] utils.write_tuning_result(params, tuning_result, exp_path + 'tuning/results.csv')
def test_accuracy_batch(self): true_label = torch.tensor([1, 0, 2, 0]).int() predictions = torch.tensor([[0.05, 0.09, 0.05], [0.1, 0.8, 0.1], [0.1, 0.2, 0.7], [0.25, 0.5, 0.25]]) acc = metrics.accuracy(predictions, true_label.long()) self.assertAlmostEqual(acc, 0.5)
def test_accuracy(self): true_label = torch.tensor(0).int() predictions = torch.tensor([0.9, 0.1]) acc = metrics.accuracy(predictions, true_label.long()) self.assertAlmostEqual(acc, 1)
def job(tuning, params_path, devices, resume, save_interval): global params if tuning: with open(params_path, 'r') as f: params = json.load(f) mode_str = 'tuning' setting = '_'.join(f'{tp}-{params[tp]}' for tp in params['tuning_params']) else: mode_str = 'train' setting = '' exp_path = os.path.join(dataset_connector.result_dir, f'{params["ex_name"]}/') os.environ['CUDA_VISIBLE_DEVICES'] = devices print("CUDA Available:", torch.cuda.is_available(), "CUDA_VISIBLE_DEVICES:", os.environ['CUDA_VISIBLE_DEVICES']) logger, writer = utils.get_logger( log_dir=exp_path + f'{mode_str}/log/{setting}', tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}') if params['augmentation'] == 'soft': params['scale_limit'] = 0.2 params['brightness_limit'] = 0.1 elif params['augmentation'] == 'middle': params['scale_limit'] = 0.3 params['shear_limit'] = 4 params['brightness_limit'] = 0.1 params['contrast_limit'] = 0.1 else: raise ValueError train_transform, eval_transform = data_utils.build_transforms( scale_limit=params['scale_limit'], shear_limit=params['shear_limit'], brightness_limit=params['brightness_limit'], contrast_limit=params['contrast_limit'], ) data_loaders = data_utils.make_train_loaders( params=params, data_root=ROOT + 'input/' + params['train_data'], use_clean_version=True, train_transform=train_transform, eval_transform=eval_transform, scale='S2', test_size=0.1, num_workers=os.cpu_count() * 2) model = models.LandmarkNet( n_classes=params['class_topk'], model_name=params['model_name'], pooling=params['pooling'], loss_module=params['loss'], s=params['s'], margin=params['margin'], theta_zero=params['theta_zero'], use_fc=params['use_fc'], fc_dim=params['fc_dim'], ).cuda() criterion = nn.CrossEntropyLoss() optimizer = utils.get_optim(params, model) # TODO: Missing initial weight file. # sdict = torch.load(ROOT + params['base_ckpt_path'])['state_dict'] # del sdict['final.weight'] # remove fully-connected layer # model.load_state_dict(sdict, strict=False) scheduler = optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=params['epochs'] * len(data_loaders['train']), eta_min=3e-6) start_epoch, end_epoch = (0, params['epochs'] - params['scaleup_epochs']) if len(devices.split(',')) > 1: model = nn.DataParallel(model) for epoch in range(start_epoch, end_epoch): logger.info(f'Epoch {epoch}/{end_epoch}') # ============================== train ============================== # model.train(True) losses = utils.AverageMeter() prec1 = utils.AverageMeter() for i, (_, x, y) in tqdm(enumerate(data_loaders['train']), total=len(data_loaders['train']), miniters=None, ncols=55): x = x.to('cuda') y = y.to('cuda') outputs = model(x, y) loss = criterion(outputs, y) optimizer.zero_grad() loss.backward() optimizer.step() scheduler.step() acc = metrics.accuracy(outputs, y) losses.update(loss.item(), x.size(0)) prec1.update(acc, x.size(0)) logger.info("Training Loss:{},Accuracy(Prec1):{}".format( loss.item(), acc)) if i % 100 == 99: logger.info( f'{epoch + i / len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}' ) train_loss = losses.avg train_acc = prec1.avg writer.add_scalars('Loss', {'train': train_loss}, epoch) writer.add_scalars('Acc', {'train': train_acc}, epoch) writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch) if (epoch + 1) == end_epoch or (epoch + 1) % save_interval == 0: output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth' utils.save_checkpoint(path=output_file_name, model=model, epoch=epoch, optimizer=optimizer, params=params) model = model.module datasets = ('oxford5k', 'paris6k', 'roxford5k', 'rparis6k') results = eval_datasets(model, datasets=datasets, ms=True, tta_gem_p=1.0, logger=logger) if tuning: tuning_result = {} for d in datasets: if d in ('oxford5k', 'paris6k'): tuning_result[d] = results[d] else: for key in ['mapE', 'mapM', 'mapH']: mapE, mapM, mapH, mpE, mpM, mpH, kappas = results[d] tuning_result[d + '-' + key] = [eval(key)] utils.write_tuning_result(params, tuning_result, exp_path + 'tuning/results.csv')