def validate_resnet(model, criterion, test_iterator, cur_epoch, dataset,
log_file):
losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter()
outputs = []
print_and_save(
'Evaluating after epoch: {} on {} set'.format(cur_epoch, dataset),
log_file)
with torch.no_grad():
model.eval()
for batch_idx, (inputs, targets,
video_names) in enumerate(test_iterator):
inputs = torch.tensor(inputs).cuda()
targets = torch.tensor(targets).cuda()
output = model(inputs)
loss = criterion(output, targets)
batch_preds = []
for j in range(output.size(0)):
res = np.argmax(output[j].detach().cpu().numpy())
label = targets[j].cpu().numpy()
outputs.append([res, label])
batch_preds.append("{}, P-L:{}-{}".format(
video_names[j], res, label))
t1, t5 = accuracy(output.detach().cpu(),
targets.cpu(),
topk=(1, 5))
top1.update(t1.item(), output.size(0))
top5.update(t5.item(), output.size(0))
losses.update(loss.item(), output.size(0))
print_and_save(
'[Batch {}/{}][Top1 {:.3f}[avg:{:.3f}], Top5 {:.3f}[avg:{:.3f}]]\n\t{}'
.format(batch_idx, len(test_iterator), top1.val, top1.avg,
top5.val, top5.avg, batch_preds), log_file)
print_and_save(
'{} Results: Loss {:.3f}, Top1 {:.3f}, Top5 {:.3f}'.format(
dataset, losses.avg, top1.avg, top5.avg), log_file)
return top1.avg, outputs
def test_cnn_do(model, criterion, test_iterator, cur_epoch, dataset, log_file,
gpus):
losses, losses_a, losses_b, top1_a, top5_a, top1_b, top5_b = AverageMeter(
), AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(
), AverageMeter(), AverageMeter()
with torch.no_grad():
model.eval()
print_and_save(
'Evaluating after epoch: {} on {} set'.format(cur_epoch, dataset),
log_file)
for batch_idx, (inputs, targets) in enumerate(test_iterator):
inputs = torch.tensor(inputs).cuda(gpus[0])
targets_a = torch.tensor(targets[0]).cuda(gpus[0])
targets_b = torch.tensor(targets[1]).cuda(gpus[0])
output_a, output_b = model(inputs)
loss_a = criterion(output_a, targets_a)
loss_b = criterion(output_b, targets_b)
loss = 0.75 * loss_a + 0.25 * loss_b
t1_a, t5_a = accuracy(output_a.detach().cpu(),
targets_a.detach().cpu(),
topk=(1, 5))
t1_b, t5_b = accuracy(output_b.detach().cpu(),
targets_b.detach().cpu(),
topk=(1, 5))
top1_a.update(t1_a.item(), output_a.size(0))
top5_a.update(t5_a.item(), output_a.size(0))
top1_b.update(t1_b.item(), output_b.size(0))
top5_b.update(t5_b.item(), output_b.size(0))
losses_a.update(loss_a.item(), output_a.size(0))
losses_b.update(loss_b.item(), output_b.size(0))
losses.update(loss.item(), output_a.size(0))
to_print = '[Epoch:{}, Batch {}/{}]' \
'[Top1_a {:.3f}[avg:{:.3f}], Top5_a {:.3f}[avg:{:.3f}],' \
'Top1_b {:.3f}[avg:{:.3f}], Top5_b {:.3f}[avg:{:.3f}]]'.format(
cur_epoch, batch_idx, len(test_iterator),
top1_a.val, top1_a.avg, top5_a.val, top5_a.avg,
top1_b.val, top1_b.avg, top5_b.val, top5_b.avg)
print_and_save(to_print, log_file)
print_and_save(
'{} Results: Loss {:.3f}, Top1_a {:.3f}, Top5_a {:.3f}, Top1_b {:.3f}, Top5_b {:.3f}'
.format(dataset, losses.avg, top1_a.avg, top5_a.avg, top1_b.avg,
top5_b.avg), log_file)
return top1_a.avg, top1_b.avg
def validate_lstm_do(model, criterion, test_iterator, cur_epoch, dataset,
log_file, args):
losses, losses_a, losses_b, top1_a, top5_a, top1_b, top5_b = AverageMeter(
), AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(
), AverageMeter(), AverageMeter()
outputs_a, outputs_b = [], []
print_and_save(
'Evaluating after epoch: {} on {} set'.format(cur_epoch, dataset),
log_file)
with torch.no_grad():
model.eval()
for batch_idx, (inputs, seq_lengths, targets,
video_names) in enumerate(test_iterator):
inputs = torch.tensor(inputs).cuda()
inputs = inputs.transpose(1, 0)
output_a, output_b = model(inputs, seq_lengths)
targets_a = torch.tensor(targets[:, 0]).cuda()
targets_b = torch.tensor(targets[:, 1]).cuda()
loss_a = criterion(output_a, targets_a)
loss_b = criterion(output_b, targets_b)
loss = 0.75 * loss_a + 0.25 * loss_b
batch_preds = []
for j in range(output_a.size(0)):
res_a = np.argmax(output_a[j].detach().cpu().numpy())
res_b = np.argmax(output_b[j].detach().cpu().numpy())
label_a = targets_a[j].cpu().numpy()
label_b = targets_b[j].cpu().numpy()
outputs_a.append([res_a, label_a])
outputs_b.append([res_b, label_b])
batch_preds.append("{}, a P-L:{}-{}, b P-L:{}-{}".format(
video_names[j], res_a, label_a, res_b, label_b))
t1_a, t5_a = accuracy(output_a.detach().cpu(),
targets_a.detach().cpu(),
topk=(1, 5))
t1_b, t5_b = accuracy(output_b.detach().cpu(),
targets_b.detach().cpu(),
topk=(1, 5))
top1_a.update(t1_a.item(), output_a.size(0))
top5_a.update(t5_a.item(), output_a.size(0))
top1_b.update(t1_b.item(), output_b.size(0))
top5_b.update(t5_b.item(), output_b.size(0))
losses_a.update(loss_a.item(), output_a.size(0))
losses_b.update(loss_b.item(), output_b.size(0))
losses.update(loss.item(), output_a.size(0))
to_print = '[Batch {}/{}]' \
'[Top1_a {:.3f}[avg:{:.3f}], Top5_a {:.3f}[avg:{:.3f}],' \
'Top1_b {:.3f}[avg:{:.3f}], Top5_b {:.3f}[avg:{:.3f}]]\n\t{}'.format(
batch_idx, len(test_iterator),
top1_a.val, top1_a.avg, top5_a.val, top5_a.avg,
top1_b.val, top1_b.avg, top5_b.val, top5_b.avg,
batch_preds)
print_and_save(to_print, log_file)
print_and_save(
'{} Results: Loss {:.3f}, Top1_a {:.3f}, Top5_a {:.3f}, Top1_b {:.3f}, Top5_b {:.3f}'
.format(dataset, losses.avg, top1_a.avg, top5_a.avg, top1_b.avg,
top5_b.avg), log_file)
return (top1_a.avg, top1_b.avg), (outputs_a, outputs_b)
def validate_lstm_attn(model, criterion, test_iterator, cur_epoch, dataset,
log_file, args):
losses, top1, top5 = AverageMeter(), AverageMeter(), AverageMeter()
predictions = []
# for attention
all_predictions = torch.zeros((0, args.lstm_seq_size, 1))
all_attentions = torch.zeros((0, args.lstm_seq_size, args.lstm_seq_size))
all_targets = torch.zeros((0, 1))
all_video_names = []
num_changing_in_seq = 0
for_the_better, for_the_worse = [], []
print_and_save(
'Evaluating after epoch: {} on {} set'.format(cur_epoch, dataset),
log_file)
with torch.no_grad():
model.eval()
for batch_idx, (inputs, seq_lengths, targets,
video_names) in enumerate(test_iterator):
inputs = torch.tensor(inputs).cuda()
targets = torch.tensor(targets).cuda()
inputs = inputs.transpose(1, 0)
outputs, attn_weights = model(inputs, seq_lengths)
loss = 0
for output in outputs:
loss += criterion(output, targets)
loss /= len(outputs)
outputs = torch.stack(outputs)
outputs = torch.argmax(outputs, dim=2).detach().cpu(
) # edw exw thn provlepsh gia kathe step tou sequence gia olo to batch
all_predictions = torch.cat(
(all_predictions, torch.transpose(outputs, 0, 1).float()),
dim=0)
outputs = outputs.numpy()
maj_vote = [
np.bincount(outputs[:, kk]).argmax()
for kk in range(len(outputs[0]))
] # to argmax kanei majority voting
for i in range(len(maj_vote)): # iteration in the batch size
if outputs[:, i].any() != outputs[-1, i]:
num_changing_in_seq += 1
if maj_vote[i] != outputs[
-1,
i]: # compare the majority vote to the prediction of the last step in the sequence
tar = targets[i].cpu().numpy()
if maj_vote[i] == tar:
for_the_better.append(video_names[i])
elif outputs[-1, i] == tar:
for_the_worse.append(video_names[i])
outputs = maj_vote
attn_weights = torch.transpose(torch.stack(attn_weights), 0,
1).detach().cpu()
all_attentions = torch.cat((all_attentions, attn_weights), dim=0)
all_targets = torch.cat(
(all_targets, targets.detach().cpu().float()), dim=0)
all_video_names = all_video_names + video_names
batch_preds = []
for j in range(len(outputs)):
res = outputs[j]
label = targets[j].cpu().numpy()
predictions.append([res, label])
batch_preds.append("{}, P-L:{}-{}".format(
video_names[j], res, label))
t1, t5 = accuracy(
output.detach().cpu(
), # use output (instead of outputs) for accuracy; outputs is used for the confusion matrix
targets.detach().cpu(),
topk=(1, 5))
top1.update(t1.item(), output.size(0))
top5.update(t5.item(), output.size(0))
losses.update(loss.item(), output.size(0))
print_and_save(
'[Batch {}/{}][Top1 {:.3f}[avg:{:.3f}], Top5 {:.3f}[avg:{:.3f}]]\n\t{}'
.format(batch_idx, len(test_iterator), top1.val, top1.avg,
top5.val, top5.avg, batch_preds), log_file)
print_and_save(
"Num samples with differences {}".format(num_changing_in_seq),
log_file)
print_and_save(
"{} changed for the better\n{}".format(len(for_the_better),
for_the_better), log_file)
print_and_save(
"{} changed for the worse\n{}".format(len(for_the_worse),
for_the_worse), log_file)
print_and_save(
'{} Results: Loss {:.3f}, Top1 {:.3f}, Top5 {:.3f}'.format(
dataset, losses.avg, top1.avg, top5.avg), log_file)
if args.save_attentions:
all_predictions = all_predictions.numpy().astype(np.int)
all_targets = all_targets.numpy().astype(np.int)
output_dir = os.path.join(os.path.dirname(log_file), "figures")
os.makedirs(output_dir, exist_ok=True)
for i in range(len(all_targets)):
name_parts = all_video_names[i].split("\\")[-2:]
output_file = os.path.join(
output_dir,
"{}_{}.png".format(name_parts[0],
name_parts[1].split('.')[0]))
showAttention(args.lstm_seq_size, all_predictions[i],
all_targets[i], all_attentions[i], output_file)
return top1.avg, predictions
def train_cnn_do(model,
optimizer,
criterion,
train_iterator,
mixup_alpha,
cur_epoch,
log_file,
gpus,
lr_scheduler=None):
batch_time, losses_a, losses_b, losses, top1_a, top5_a, top1_b, top5_b = AverageMeter(
), AverageMeter(), AverageMeter(), AverageMeter(), AverageMeter(
), AverageMeter(), AverageMeter(), AverageMeter()
model.train()
if not isinstance(lr_scheduler, CyclicLR):
lr_scheduler.step()
print_and_save('*********', log_file)
print_and_save('Beginning of epoch: {}'.format(cur_epoch), log_file)
t0 = time.time()
for batch_idx, (inputs, targets) in enumerate(train_iterator):
if isinstance(lr_scheduler, CyclicLR):
lr_scheduler.step()
inputs = torch.tensor(inputs, requires_grad=True).cuda(gpus[0])
targets_a = torch.tensor(targets[0]).cuda(gpus[0])
targets_b = torch.tensor(targets[1]).cuda(gpus[0])
output_a, output_b = model(inputs)
loss_a = criterion(output_a, targets_a)
loss_b = criterion(output_b, targets_b)
loss = 0.75 * loss_a + 0.25 * loss_b
optimizer.zero_grad()
loss.backward()
optimizer.step()
t1_a, t5_a = accuracy(output_a.detach().cpu(),
targets_a.detach().cpu(),
topk=(1, 5))
t1_b, t5_b = accuracy(output_b.detach().cpu(),
targets_b.detach().cpu(),
topk=(1, 5))
top1_a.update(t1_a.item(), output_a.size(0))
top5_a.update(t5_a.item(), output_a.size(0))
top1_b.update(t1_b.item(), output_b.size(0))
top5_b.update(t5_b.item(), output_b.size(0))
losses_a.update(loss_a.item(), output_a.size(0))
losses_b.update(loss_b.item(), output_b.size(0))
losses.update(loss.item(), output_a.size(0))
batch_time.update(time.time() - t0)
t0 = time.time()
to_print = '[Epoch:{}, Batch {}/{} in {:.3f} s]'\
'[Losses {:.4f}[avg:{:.4f}], loss_a {:.4f}[avg:{:.4f}], loss_b {:.4f}[avg:{:.4f}],' \
'Top1_a {:.3f}[avg:{:.3f}], Top5_a {:.3f}[avg:{:.3f}],' \
'Top1_b {:.3f}[avg:{:.3f}], Top5_b {:.3f}[avg:{:.3f}]],' \
'LR {:.6f}'.format(
cur_epoch, batch_idx, len(train_iterator), batch_time.val,
losses_a.val, losses_a.avg, losses_b.val, losses_b.avg, losses.val, losses.avg,
top1_a.val, top1_a.avg, top5_a.val, top5_a.avg,
top1_b.val, top1_b.avg, top5_b.val, top5_b.avg,
lr_scheduler.get_lr()[0])
print_and_save(to_print, log_file)
def train_cnn(model,
optimizer,
criterion,
train_iterator,
mixup_alpha,
cur_epoch,
log_file,
gpus,
lr_scheduler=None):
batch_time, losses, top1, top5 = AverageMeter(), AverageMeter(
), AverageMeter(), AverageMeter()
model.train()
if not isinstance(lr_scheduler, CyclicLR):
lr_scheduler.step()
print_and_save('*********', log_file)
print_and_save('Beginning of epoch: {}'.format(cur_epoch), log_file)
t0 = time.time()
for batch_idx, (inputs, targets) in enumerate(train_iterator):
if isinstance(lr_scheduler, CyclicLR):
lr_scheduler.step()
inputs = inputs.cuda(gpus[0])
targets = targets.cuda(gpus[0])
# TODO: Fix mixup and cuda integration, especially for mfnet
if mixup_alpha != 1:
inputs, targets_a, targets_b, lam = mixup_data(
inputs, targets, mixup_alpha)
output = model(inputs)
if mixup_alpha != 1:
loss = mixup_criterion(criterion, output, targets_a, targets_b,
lam)
else:
loss = criterion(output, targets)
optimizer.zero_grad()
loss.backward()
# if clip_gradient is not None:
# total_norm = torch.nn.clip_grad_norm_(model.parameters(), clip_gradient)
# if total_norm > clip_gradient:
# to_print = "clipping gradient: {} with coef {}".format(total_norm, clip_gradient / total_norm)
# print_and_save(to_print, log_file)
optimizer.step()
t1, t5 = accuracy(output.detach(), targets.detach(), topk=(1, 5))
top1.update(t1.item(), output.size(0))
top5.update(t5.item(), output.size(0))
losses.update(loss.item(), output.size(0))
batch_time.update(time.time() - t0)
t0 = time.time()
print_and_save(
'[Epoch:{}, Batch {}/{} in {:.3f} s][Loss {:.4f}[avg:{:.4f}], Top1 {:.3f}[avg:{:.3f}], Top5 {:.3f}[avg:{:.3f}]], LR {:.6f}'
.format(cur_epoch, batch_idx, len(train_iterator), batch_time.val,
losses.val, losses.avg, top1.val, top1.avg, top5.val,
top5.avg,
lr_scheduler.get_lr()[0]), log_file)
print_and_save("Epoch train time: {}".format(batch_time.sum), log_file)