def get_accuracy_score(targets, predictions):
assert len(targets) == len(predictions)
met = []
for i in range(len(targets)):
met.append(accuracy(predictions[i], targets[i]).item())
return np.mean(met)
def test_accuracy():
x = torch.rand(8, 10)
y = torch.rand(8).long()
acc = metrics.accuracy(x, y)
assert type(acc) is float
assert 0.0 <= acc <= 1.0
def ood_data_experiment():
"""Repeat experiment from Ovidia et al. (2019), plotting boxplots with accuracy and ece on corrupted data"""
data_dir = "../../dataloaders/data/"
model_list = ["distilled", "dirichlet_distill", "mixture_distill", "ensemble_new", "vanilla", "temp_scaling",
"dropout_nofirst", "ll_dropout", "svi", "ll_svi"]
corruption_list = ["brightness", "contrast", "defocus_blur", "elastic_transform", "fog", "frost",
"gaussian_blur", "gaussian_noise", "glass_blur", "impulse_noise", "pixelate",
"saturate", "shot_noise", "spatter", "speckle_noise", "zoom_blur"]
intensity_list = [0, 1, 2, 3, 4, 5]
rep_list = [1, 2, 3, 4, 5]
ensemble_inds = np.load("data/ensemble_indices.npy")
ensemble_size = 10
acc_list = []
ece_list = []
for intensity in intensity_list:
print(intensity)
acc = np.zeros((len(corruption_list), len(rep_list), len(model_list)))
ece = np.zeros((len(corruption_list), len(rep_list), len(model_list)))
for i, corruption in enumerate(corruption_list):
print(corruption)
for j, model in enumerate(model_list):
for k, rep in enumerate(rep_list):
if model == "ensemble_new":
inds = ensemble_inds[((rep-1) * ensemble_size):(rep * ensemble_size)]
else:
inds = None
data = cifar10_benchmark_model_predictions.Cifar10DataPredictions(model=model,
corruption=corruption,
intensity=intensity,
data_dir=data_dir,
rep=rep,
ensemble_indices=inds)
if model in ["distilled", "ensemble_new", "dirichlet_distill"]:
data.set.predictions = np.mean(data.set.predictions, axis=1)
acc[i, k-1, j] = metrics.accuracy(torch.tensor(data.set.predictions),
torch.tensor(data.set.targets, dtype=torch.long))
ece[i, k-1, j] = metrics.ece(data.set.predictions, data.set.targets)
acc_list.append(acc.reshape(-1, acc.shape[-1]))
ece_list.append(ece.reshape(-1, ece.shape[-1]))
model_list_text = ["Gaussian Distilled", "Dirichlet Distilled", "Mixture Distilled", "Ensemble", "Vanilla",
"Temp Scaling", "Dropout", "LL Dropout", "SVI", "LL SVI"]
colors = ["#A6CEE3", "#1F78B4", "#B2DF8A", "#33A02C", "#FB9A99", "#E31A1C", "#FDBF6F", "#FF7F00", "#CAB2D6",
"#6A3D9A"]
make_boxplot(acc_list, "data/fig/acc_reduced_benchmark_experiments.tikz", model_list=model_list_text, colors=colors)
make_boxplot(ece_list, "data/fig/ece_reduced_benchmark_experiments.tikz", label="ECE", model_list=model_list_text,
colors=colors, max_y=0.8)
def evaluate(model, data_input, gold_output):
predictions = model.predict(data_input, batch_size=config.Params.batch_size, verbose=1)
if len(predictions.shape) == 3:
predictions_classes = np.argmax(predictions, axis=2)
train_batch_f1 = metrics.accuracy_per_sentence(predictions_classes, gold_output)
print("Results (per sentence): ", train_batch_f1)
train_y_properties_stream = gold_output.reshape(gold_output.shape[0] * gold_output.shape[1])
predictions_classes = predictions_classes.reshape(predictions_classes.shape[0] * predictions_classes.shape[1])
class_mask = train_y_properties_stream != 0
train_y_properties_stream = train_y_properties_stream[class_mask]
predictions_classes = predictions_classes[class_mask]
else:
predictions_classes = np.argmax(predictions, axis=1)
train_y_properties_stream = gold_output
accuracy = metrics.accuracy(predictions_classes, train_y_properties_stream)
micro_scores = metrics.compute_micro_PRF(predictions_classes, train_y_properties_stream,
empty_label=keras_models.p0_index)
print("Results: Accuracy: ", accuracy)
print("Results: Micro-Average F1: ", micro_scores)
return predictions_classes, predictions
def evaluate(self, model, device):
model.eval()
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for input_ids, input_mask, segment_ids, label_ids in tqdm(
self.evaluation_data_loader, desc="Evaluating"):
input_ids = input_ids.to(device)
input_mask = input_mask.to(device)
segment_ids = segment_ids.to(device)
label_ids = label_ids.to(device)
with torch.no_grad():
tmp_eval_loss = model(input_ids, segment_ids, input_mask,
label_ids)
logits = model(input_ids, segment_ids, input_mask)
logits = logits.detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
tmp_eval_accuracy = accuracy(logits, label_ids)
eval_loss += tmp_eval_loss.mean().item()
eval_accuracy += tmp_eval_accuracy
nb_eval_examples += input_ids.size(0)
nb_eval_steps += 1
eval_loss = eval_loss / nb_eval_steps
eval_accuracy = eval_accuracy / nb_eval_examples
result = {'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy}
with open(self.output_path, "a+") as writer:
tqdm.write("***** Eval results *****")
writer.write("***** Eval results *****\n")
for key in sorted(result.keys()):
tqdm.write("{}: {}".format(key, str(result[key])))
writer.write("{}: {}\n".format(key, str(result[key])))
def build_model():
"""Скрипт для потсроения модели"""
print('reading/preparing train data')
raw_data = read_csv_data(train_csv_path)
assert all([f in raw_data.columns.tolist() for f in feats
]), 'not all feature columns are present in train data'
assert 'y' in raw_data, 'target column is not present in data'
assert raw_data.shape[0] > N_NEIGHBORS
X = raw_data[feats].values
y = raw_data['y'].values
assert not (X == X[0]).all(), 'all rows in training data are the same'
assert np.unique(
y).shape[0] > 1, 'target data must contain more than 1 unique value'
X_train, y_train, X_val, y_val = train_test_split(X, y, 0.2)
print('training model')
mdl = ModelPipeline(steps)
mdl.fit(X_train, y_train)
y_pred = mdl.predict(X_val)
acc = accuracy(y_pred, y_val)
print(f'validation accuracy: {acc}')
print('saving model')
if not os.path.exists(saved_models_path):
os.makedirs(saved_models_path)
pickle.dump(mdl, open(os.path.join(saved_models_path, 'mdl.p'), 'wb'))
print('done')
def job(tuning, params_path, devices, resume, save_interval):
global params
if tuning:
with open(params_path, 'r') as f:
params = json.load(f)
mode_str = 'tuning'
setting = '_'.join(f'{tp}-{params[tp]}'
for tp in params['tuning_params'])
else:
mode_str = 'train'
setting = ''
# パラメーターを変えるときにseedも変えたい(seed averagingの効果を期待)
seed = sum(ord(_) for _ in str(params.values()))
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.benchmark = False
exp_path = ROOT + f'experiments/{params["ex_name"]}/'
os.environ['CUDA_VISIBLE_DEVICES'] = devices
logger, writer = utils.get_logger(
log_dir=exp_path + f'{mode_str}/log/{setting}',
tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}')
if params['augmentation'] == 'soft':
params['scale_limit'] = 0.2
params['brightness_limit'] = 0.1
elif params['augmentation'] == 'middle':
params['scale_limit'] = 0.3
params['shear_limit'] = 4
params['brightness_limit'] = 0.1
params['contrast_limit'] = 0.1
else:
raise ValueError
train_transform, eval_transform = data_utils.build_transforms(
scale_limit=params['scale_limit'],
shear_limit=params['shear_limit'],
brightness_limit=params['brightness_limit'],
contrast_limit=params['contrast_limit'],
)
data_loaders = data_utils.make_train_loaders(
params=params,
data_root=ROOT + 'input/' + params['data'],
train_transform=train_transform,
eval_transform=eval_transform,
scale='S',
test_size=0,
class_topk=params['class_topk'],
num_workers=8)
model = models.LandmarkNet(
n_classes=params['class_topk'],
model_name=params['model_name'],
pooling=params['pooling'],
loss_module=params['loss'],
s=params['s'],
margin=params['margin'],
theta_zero=params['theta_zero'],
use_fc=params['use_fc'],
fc_dim=params['fc_dim'],
).cuda()
optimizer = utils.get_optim(params, model)
criterion = nn.CrossEntropyLoss()
scheduler = optim.lr_scheduler.CosineAnnealingLR(
optimizer,
T_max=params['epochs'] * len(data_loaders['train']),
eta_min=3e-6)
start_epoch = 0
if len(devices.split(',')) > 1:
model = nn.DataParallel(model)
for epoch in range(start_epoch, params['epochs']):
logger.info(
f'Epoch {epoch}/{params["epochs"]} | lr: {optimizer.param_groups[0]["lr"]}'
)
# ============================== train ============================== #
model.train(True)
losses = utils.AverageMeter()
prec1 = utils.AverageMeter()
for i, (_, x, y) in tqdm(enumerate(data_loaders['train']),
total=len(data_loaders['train']),
miniters=None,
ncols=55):
x = x.to('cuda')
y = y.to('cuda')
outputs = model(x, y)
loss = criterion(outputs, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
acc = metrics.accuracy(outputs, y)
losses.update(loss.item(), x.size(0))
prec1.update(acc, x.size(0))
if i % 100 == 99:
logger.info(
f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}'
)
train_loss = losses.avg
train_acc = prec1.avg
writer.add_scalars('Loss', {'train': train_loss}, epoch)
writer.add_scalars('Acc', {'train': train_acc}, epoch)
writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch)
if (epoch + 1) == params['epochs'] or (epoch + 1) % save_interval == 0:
output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth'
utils.save_checkpoint(path=output_file_name,
model=model,
epoch=epoch,
optimizer=optimizer,
params=params)
model = model.module
datasets = ('roxford5k', 'rparis6k')
results = eval_datasets(model,
datasets=datasets,
ms=False,
tta_gem_p=1.0,
logger=logger)
if tuning:
tuning_result = {}
for d in datasets:
for key in ['mapE', 'mapM', 'mapH']:
mapE, mapM, mapH, mpE, mpM, mpH, kappas = results[d]
tuning_result[d + '-' + key] = [eval(key)]
utils.write_tuning_result(params, tuning_result,
exp_path + 'tuning/results.csv')
total=len(data_loaders['train']),
miniters=None, ncols=55):
if num_GPU>0:
x = x.to('cuda')
y = y.to('cuda')
outputs = model(x, y)
loss = criterion(outputs, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
acc = metrics.accuracy(outputs, y)
losses.update(loss.item(), x.size(0))
prec1.update(acc, x.size(0))
if i % 100 == 99:
logger.info(f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}')
train_loss = losses.avg
train_acc = prec1.avg
print("[{:5d}] => loss={:.9f}, acc={:.9f}; lr={:.9f}.".format(epoch, train_loss, train_acc))
writer.add_scalars('Loss', {'train': train_loss}, epoch)
writer.add_scalars('Acc', {'train': train_acc}, epoch)
writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch)
def train_distilled_network_dirichlet(
model_dir="models/distilled_model_cifar10_dirichlet"):
"""Distill ensemble with distribution distillation (Dirichlet) """
args = utils.parse_args()
log_file = Path("{}.log".format(datetime.now().strftime('%Y%m%d_%H%M%S')))
utils.setup_logger(log_path=Path.cwd() / args.log_dir / log_file,
log_level=args.log_level)
data_ind = np.load(
"src/experiments/cifar10/training_files/training_data_indices.npy")
num_train_points = 40000
train_ind = data_ind[:num_train_points]
valid_ind = data_ind[num_train_points:]
train_data = cifar10_ensemble_pred.Cifar10Data(ind=train_ind,
augmentation=True)
valid_data = cifar10_ensemble_pred.Cifar10Data(ind=valid_ind)
train_loader = torch.utils.data.DataLoader(train_data.set,
batch_size=100,
shuffle=True,
num_workers=0)
valid_loader = torch.utils.data.DataLoader(valid_data.set,
batch_size=100,
shuffle=True,
num_workers=0)
test_data = cifar10_ensemble_pred.Cifar10Data(train=False)
test_loader = torch.utils.data.DataLoader(test_data.set,
batch_size=64,
shuffle=True,
num_workers=0)
ensemble_size = 10
# Note that the ensemble predictions are assumed to have been saved to file (see ensemble_predictions.py),
# ensemble_indices.npy contains the order of the ensemble members such that ind[:ensemble_size] are the indices
# of the first ensemble, ind[ensemble_size:2*ensemble_size] are the indices of the second ensemble and so on
ind = np.load("src/experiments/cifar10/training_files/ensemble_indices.npy"
)[((args.rep - 1) * ensemble_size):(args.rep *
ensemble_size)]
ensemble = ensemble_wrapper.EnsembleWrapper(output_size=10, indices=ind)
device = utils.torch_settings(args.seed, args.gpu)
distilled_model = cifar_resnet_dirichlet.CifarResnetDirichlet(
ensemble,
resnet_utils.BasicBlock, [3, 2, 2, 2],
device=device,
learning_rate=args.lr)
loss_metric = metrics.Metric(name="Mean loss",
function=distilled_model.calculate_loss)
distilled_model.add_metric(loss_metric)
distilled_model.train(train_loader,
num_epochs=args.num_epochs,
validation_loader=valid_loader)
distilled_model.eval_mode()
predicted_distribution = []
all_labels = []
for batch in test_loader:
inputs, labels = batch
inputs, labels = inputs[0].to(distilled_model.device), labels.to(
distilled_model.device)
predicted_distribution.append(
distilled_model.predict(inputs).to(distilled_model.device))
all_labels.append(labels.long())
test_acc = metrics.accuracy(torch.cat(predicted_distribution),
torch.cat(all_labels))
LOGGER.info("Test accuracy is {}".format(test_acc))
torch.save(distilled_model.state_dict(), model_dir)
def job(tuning, params_path, devices, resume, save_interval):
global params
if tuning:
with open(params_path, 'r') as f:
params = json.load(f)
mode_str = 'tuning'
setting = '_'.join(f'{tp}-{params[tp]}'
for tp in params['tuning_params'])
else:
mode_str = 'train'
setting = ''
exp_path = ROOT + f'experiments/{params["ex_name"]}/'
os.environ['CUDA_VISIBLE_DEVICES'] = devices
if resume is None:
# C-AIRとABCIで整合性が取れるようにしている。
params[
'base_ckpt_path'] = f'experiments/v1only/ep4_augmentation-soft_epochs-5_loss-{params["loss"]}.pth'
params[
'clean_path'] = ROOT + f'input/clean/train19_cleaned_verifythresh{params["verifythresh"]}_freqthresh{params["freqthresh"]}.csv'
else:
params = utils.load_checkpoint(path=resume, params=True)['params']
logger, writer = utils.get_logger(
log_dir=exp_path + f'{mode_str}/log/{setting}',
tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}')
if params['augmentation'] == 'soft':
params['scale_limit'] = 0.2
params['brightness_limit'] = 0.1
elif params['augmentation'] == 'middle':
params['scale_limit'] = 0.3
params['shear_limit'] = 4
params['brightness_limit'] = 0.1
params['contrast_limit'] = 0.1
else:
raise ValueError
train_transform, eval_transform = data_utils.build_transforms(
scale_limit=params['scale_limit'],
shear_limit=params['shear_limit'],
brightness_limit=params['brightness_limit'],
contrast_limit=params['contrast_limit'],
)
data_loaders = data_utils.make_train_loaders(
params=params,
data_root=ROOT + 'input/' + params['data'],
train_transform=train_transform,
eval_transform=eval_transform,
scale='SS2',
test_size=0,
class_topk=params['class_topk'],
num_workers=8)
model = models.LandmarkNet(
n_classes=params['class_topk'],
model_name=params['model_name'],
pooling=params['pooling'],
loss_module=params['loss'],
s=params['s'],
margin=params['margin'],
theta_zero=params['theta_zero'],
use_fc=params['use_fc'],
fc_dim=params['fc_dim'],
).cuda()
criterion = nn.CrossEntropyLoss()
optimizer = utils.get_optim(params, model)
if resume is None:
sdict = torch.load(ROOT + params['base_ckpt_path'])['state_dict']
if params['loss'] == 'adacos':
del sdict['final.W'] # remove fully-connected layer
elif params['loss'] == 'softmax':
del sdict['final.weight'], sdict[
'final.bias'] # remove fully-connected layer
else:
del sdict['final.weight'] # remove fully-connected layer
model.load_state_dict(sdict, strict=False)
scheduler = optim.lr_scheduler.CosineAnnealingLR(
optimizer,
T_max=params['epochs'] * len(data_loaders['train']),
eta_min=3e-6)
start_epoch, end_epoch = (0,
params['epochs'] - params['scaleup_epochs'])
else:
ckpt = utils.load_checkpoint(path=resume,
model=model,
optimizer=optimizer,
epoch=True)
model, optimizer, start_epoch = ckpt['model'], ckpt[
'optimizer'], ckpt['epoch'] + 1
end_epoch = params['epochs']
scheduler = optim.lr_scheduler.CosineAnnealingLR(
optimizer,
T_max=params['epochs'] * len(data_loaders['train']),
eta_min=3e-6,
last_epoch=start_epoch * len(data_loaders['train']))
setting += 'scaleup_' + resume.split('/')[-1].replace('.pth', '')
data_loaders = data_utils.make_verified_train_loaders(
params=params,
data_root=ROOT + 'input/' + params['data'],
train_transform=train_transform,
eval_transform=eval_transform,
scale='M2',
test_size=0,
num_workers=8)
batch_norm.freeze_bn(model)
if len(devices.split(',')) > 1:
model = nn.DataParallel(model)
for epoch in range(start_epoch, end_epoch):
logger.info(f'Epoch {epoch}/{end_epoch}')
# ============================== train ============================== #
model.train(True)
losses = utils.AverageMeter()
prec1 = utils.AverageMeter()
for i, (_, x, y) in tqdm(enumerate(data_loaders['train']),
total=len(data_loaders['train']),
miniters=None,
ncols=55):
x = x.to('cuda')
y = y.to('cuda')
outputs = model(x, y)
loss = criterion(outputs, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
acc = metrics.accuracy(outputs, y)
losses.update(loss.item(), x.size(0))
prec1.update(acc, x.size(0))
if i % 100 == 99:
logger.info(
f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}'
)
train_loss = losses.avg
train_acc = prec1.avg
writer.add_scalars('Loss', {'train': train_loss}, epoch)
writer.add_scalars('Acc', {'train': train_acc}, epoch)
writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch)
if (epoch + 1) == end_epoch or (epoch + 1) % save_interval == 0:
output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth'
utils.save_checkpoint(path=output_file_name,
model=model,
epoch=epoch,
optimizer=optimizer,
params=params)
model = model.module
datasets = ('oxford5k', 'paris6k', 'roxford5k', 'rparis6k')
results = eval_datasets(model,
datasets=datasets,
ms=True,
tta_gem_p=1.0,
logger=logger)
if tuning:
tuning_result = {}
for d in datasets:
if d in ('oxford5k', 'paris6k'):
tuning_result[d] = results[d]
else:
for key in ['mapE', 'mapM', 'mapH']:
mapE, mapM, mapH, mpE, mpM, mpH, kappas = results[d]
tuning_result[d + '-' + key] = [eval(key)]
utils.write_tuning_result(params, tuning_result,
exp_path + 'tuning/results.csv')
def job(tuning, params_path, devices, resume, save_interval):
global params
if tuning:
with open(params_path, 'r') as f:
params = json.load(f)
mode_str = 'tuning'
setting = '_'.join(f'{tp}-{params[tp]}'
for tp in params['tuning_params'])
else:
mode_str = 'train'
setting = ''
exp_path = ROOT + f'experiments/{params["ex_name"]}/'
os.environ['CUDA_VISIBLE_DEVICES'] = devices
logger, writer = utils.get_logger(
log_dir=exp_path + f'{mode_str}/log/{setting}',
tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}')
train_transform, eval_transform = build_transforms(
scale_range=params['scale_range'],
brightness_range=params['brightness_range'])
data_loaders = data_utils.make_train_loaders(
params=params,
data_root=ROOT + 'input/train2018',
train_transform=train_transform,
eval_transform=eval_transform,
class_topk=params['class_topk'],
num_workers=8)
model = models.LandmarkFishNet(
n_classes=params['class_topk'],
model_name=params['model_name'],
pooling_strings=params['pooling'].split(','),
loss_module='arcface',
s=30.0,
margin=params['margin'],
use_fc=params['use_fc'],
fc_dim=params['fc_dim'],
).cuda()
optimizer = utils.get_optim(params, model)
criterion = nn.CrossEntropyLoss()
scheduler = optim.lr_scheduler.CosineAnnealingLR(
optimizer,
T_max=params['epochs'] * len(data_loaders['train']),
eta_min=1e-6)
if len(devices.split(',')) > 1:
model = nn.DataParallel(model)
if resume is not None:
model, optimizer = utils.load_checkpoint(path=resume,
model=model,
optimizer=optimizer)
for epoch in range(params['epochs']):
logger.info(
f'Epoch {epoch}/{params["epochs"]} | lr: {optimizer.param_groups[0]["lr"]}'
)
# ============================== train ============================== #
model.train(True)
losses = utils.AverageMeter()
prec1 = utils.AverageMeter()
for i, (_, x, y) in tqdm(enumerate(data_loaders['train']),
total=len(data_loaders['train']),
miniters=None,
ncols=55):
x = x.to('cuda')
y = y.to('cuda')
outputs = model(x, y)
loss = criterion(outputs, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
acc = metrics.accuracy(outputs, y)
losses.update(loss.item(), x.size(0))
prec1.update(acc, x.size(0))
if i % 100 == 99:
logger.info(
f'{epoch+i/len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}'
)
train_loss = losses.avg
train_acc = prec1.avg
# ============================== validation ============================== #
model.train(False)
losses.reset()
prec1.reset()
for i, (_, x, y) in tqdm(enumerate(data_loaders['val']),
total=len(data_loaders['val']),
miniters=None,
ncols=55):
x = x.to('cuda')
y = y.to('cuda')
with torch.no_grad():
outputs = model(x, y)
loss = criterion(outputs, y)
acc = metrics.accuracy(outputs, y)
losses.update(loss.item(), x.size(0))
prec1.update(acc, x.size(0))
val_loss = losses.avg
val_acc = prec1.avg
logger.info(f'[Val] Loss: \033[1m{val_loss:.4f}\033[0m | '
f'Acc: \033[1m{val_acc:.4f}\033[0m\n')
writer.add_scalars('Loss', {'train': train_loss}, epoch)
writer.add_scalars('Acc', {'train': train_acc}, epoch)
writer.add_scalars('Loss', {'val': val_loss}, epoch)
writer.add_scalars('Acc', {'val': val_acc}, epoch)
writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch)
if save_interval > 0:
if (epoch +
1) == params['epochs'] or (epoch + 1) % save_interval == 0:
output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth'
utils.save_checkpoint(path=output_file_name,
model=model,
epoch=epoch,
optimizer=optimizer,
params=params)
if tuning:
tuning_result = {}
for key in ['train_loss', 'train_acc', 'val_loss', 'val_acc']:
tuning_result[key] = [eval(key)]
utils.write_tuning_result(params, tuning_result,
exp_path + 'tuning/results.csv')
def test_accuracy_batch(self):
true_label = torch.tensor([1, 0, 2, 0]).int()
predictions = torch.tensor([[0.05, 0.09, 0.05], [0.1, 0.8, 0.1],
[0.1, 0.2, 0.7], [0.25, 0.5, 0.25]])
acc = metrics.accuracy(predictions, true_label.long())
self.assertAlmostEqual(acc, 0.5)
def test_accuracy(self):
true_label = torch.tensor(0).int()
predictions = torch.tensor([0.9, 0.1])
acc = metrics.accuracy(predictions, true_label.long())
self.assertAlmostEqual(acc, 1)
def job(tuning, params_path, devices, resume, save_interval):
global params
if tuning:
with open(params_path, 'r') as f:
params = json.load(f)
mode_str = 'tuning'
setting = '_'.join(f'{tp}-{params[tp]}'
for tp in params['tuning_params'])
else:
mode_str = 'train'
setting = ''
exp_path = os.path.join(dataset_connector.result_dir,
f'{params["ex_name"]}/')
os.environ['CUDA_VISIBLE_DEVICES'] = devices
print("CUDA Available:", torch.cuda.is_available(),
"CUDA_VISIBLE_DEVICES:", os.environ['CUDA_VISIBLE_DEVICES'])
logger, writer = utils.get_logger(
log_dir=exp_path + f'{mode_str}/log/{setting}',
tensorboard_dir=exp_path + f'{mode_str}/tf_board/{setting}')
if params['augmentation'] == 'soft':
params['scale_limit'] = 0.2
params['brightness_limit'] = 0.1
elif params['augmentation'] == 'middle':
params['scale_limit'] = 0.3
params['shear_limit'] = 4
params['brightness_limit'] = 0.1
params['contrast_limit'] = 0.1
else:
raise ValueError
train_transform, eval_transform = data_utils.build_transforms(
scale_limit=params['scale_limit'],
shear_limit=params['shear_limit'],
brightness_limit=params['brightness_limit'],
contrast_limit=params['contrast_limit'],
)
data_loaders = data_utils.make_train_loaders(
params=params,
data_root=ROOT + 'input/' + params['train_data'],
use_clean_version=True,
train_transform=train_transform,
eval_transform=eval_transform,
scale='S2',
test_size=0.1,
num_workers=os.cpu_count() * 2)
model = models.LandmarkNet(
n_classes=params['class_topk'],
model_name=params['model_name'],
pooling=params['pooling'],
loss_module=params['loss'],
s=params['s'],
margin=params['margin'],
theta_zero=params['theta_zero'],
use_fc=params['use_fc'],
fc_dim=params['fc_dim'],
).cuda()
criterion = nn.CrossEntropyLoss()
optimizer = utils.get_optim(params, model)
# TODO: Missing initial weight file.
# sdict = torch.load(ROOT + params['base_ckpt_path'])['state_dict']
# del sdict['final.weight'] # remove fully-connected layer
# model.load_state_dict(sdict, strict=False)
scheduler = optim.lr_scheduler.CosineAnnealingLR(
optimizer,
T_max=params['epochs'] * len(data_loaders['train']),
eta_min=3e-6)
start_epoch, end_epoch = (0, params['epochs'] - params['scaleup_epochs'])
if len(devices.split(',')) > 1:
model = nn.DataParallel(model)
for epoch in range(start_epoch, end_epoch):
logger.info(f'Epoch {epoch}/{end_epoch}')
# ============================== train ============================== #
model.train(True)
losses = utils.AverageMeter()
prec1 = utils.AverageMeter()
for i, (_, x, y) in tqdm(enumerate(data_loaders['train']),
total=len(data_loaders['train']),
miniters=None,
ncols=55):
x = x.to('cuda')
y = y.to('cuda')
outputs = model(x, y)
loss = criterion(outputs, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
acc = metrics.accuracy(outputs, y)
losses.update(loss.item(), x.size(0))
prec1.update(acc, x.size(0))
logger.info("Training Loss:{},Accuracy(Prec1):{}".format(
loss.item(), acc))
if i % 100 == 99:
logger.info(
f'{epoch + i / len(data_loaders["train"]):.2f}epoch | {setting} acc: {prec1.avg}'
)
train_loss = losses.avg
train_acc = prec1.avg
writer.add_scalars('Loss', {'train': train_loss}, epoch)
writer.add_scalars('Acc', {'train': train_acc}, epoch)
writer.add_scalar('LR', optimizer.param_groups[0]['lr'], epoch)
if (epoch + 1) == end_epoch or (epoch + 1) % save_interval == 0:
output_file_name = exp_path + f'ep{epoch}_' + setting + '.pth'
utils.save_checkpoint(path=output_file_name,
model=model,
epoch=epoch,
optimizer=optimizer,
params=params)
model = model.module
datasets = ('oxford5k', 'paris6k', 'roxford5k', 'rparis6k')
results = eval_datasets(model,
datasets=datasets,
ms=True,
tta_gem_p=1.0,
logger=logger)
if tuning:
tuning_result = {}
for d in datasets:
if d in ('oxford5k', 'paris6k'):
tuning_result[d] = results[d]
else:
for key in ['mapE', 'mapM', 'mapH']:
mapE, mapM, mapH, mpE, mpM, mpH, kappas = results[d]
tuning_result[d + '-' + key] = [eval(key)]
utils.write_tuning_result(params, tuning_result,
exp_path + 'tuning/results.csv')
