def validation(model, val_dataloader, current_epoch, criterion, writer, run_type='val',model_dir='./checkpoint'):
batch_time = AverageMeter()
losses = AverageMeter()
model.eval()
progbar = Progbar(len(val_dataloader.dataset), stateful_metrics=['run-type'])
end = time.time()
with torch.no_grad():
probs = np.array([])
gt_labels = np.array([])
names = []
for step, (labels, imgs, img_paths) in enumerate(val_dataloader):
names.extend(img_paths)
imgs = imgs.cuda()
outputs = model(imgs).reshape(-1).cpu()
loss = criterion(outputs,labels.float())
losses.update(loss)
outputs = torch.sigmoid(outputs).cpu().numpy().reshape(-1)
probs = np.concatenate((probs, outputs), axis=0)
gt_labels = np.concatenate((gt_labels, labels.data.numpy().reshape(-1)), axis=0)
assert gt_labels.shape == probs.shape
progbar.add(imgs.size(0),
values=[('run-type', run_type), ('val_loss', loss.item())]) # ,('batch_time', batch_time.val)])
batch_time.update(time.time() - end)
end = time.time()
metrixs = evaluate(gt_labels, probs > 0.5, probs)
writer.add_scalar('val_loss_epoch', losses.avg, current_epoch)
writer.add_scalar('val_f1',metrixs['f1'])
print("Epoch: {} f1: {}".format(current_epoch, metrixs['f1']))
torch.save({
'epoch': current_epoch + 1,
'state_dict': model.state_dict(),
}, model_dir + '/model_last.pth.tar')
def prec_bn(train_loader, model, epoch, args, config):
"""precise batch norm"""
# set up meters
batch_time = AverageMeter()
# number of iterations per epoch
num_iters = len(train_loader)
# switch to train mode & reduce batch norm momentum
model.train()
for m in model.modules():
if isinstance(m, nn.BatchNorm3d):
m.momentum = 0.05
# data prefetcher with noramlization
train_loader = ClipPrefetcherJoint(train_loader, config['input']['mean'],
config['input']['std'])
# main loop
end = time.time()
input, _ = train_loader.next()
i = 0
while input is not None:
# input & target are pre-fetched
i += 1
# no gradient to params
with torch.no_grad():
_ = model(input)
# printing (on the first GPU)
if (i % (args.print_freq * 2)) == 0 and (args.local_rank == 0):
# measure elapsed time
torch.cuda.synchronize()
batch_time.update((time.time() - end) / (args.print_freq * 2))
end = time.time()
print('Prec BN: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'.format(
epoch + 1, i, num_iters, batch_time=batch_time))
# prefetch next batch
input, _ = train_loader.next()
# reset bn params back
for m in model.modules():
if isinstance(m, nn.BatchNorm3d):
m.momentum = 0.1
return
def test(data_loader, model, opt, class_names):
print('test')
model.eval()
# pytroch version check
torch_version = float(torch.__version__[:3])
batch_time = AverageMeter()
data_time = AverageMeter()
end_time = time.time()
output_buffer = []
previous_video_id = ''
test_results = {'results': {}}
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
inputs = Variable(inputs, volatile=True)
outputs = model(inputs)
if not opt.no_softmax_in_test:
outputs = F.softmax(outputs)
for j in range(outputs.size(0)):
if not (i == 0 and j == 0) and targets[j] != previous_video_id:
if torch_version < 0.4:
calculate_video_results(output_buffer, previous_video_id,
test_results, class_names)
else:
calculate_video_results_pt_0_4(output_buffer,
previous_video_id,
test_results, class_names)
output_buffer = []
output_buffer.append(outputs[j].data.cpu())
previous_video_id = targets[j]
if (i % 100) == 0:
with open(
os.path.join(opt.result_path,
'{}.json'.format(opt.test_subset)), 'w') as f:
json.dump(test_results, f)
batch_time.update(time.time() - end_time)
end_time = time.time()
print('[{}/{}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format(
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time))
with open(os.path.join(opt.result_path, '{}.json'.format(opt.test_subset)),
'w') as f:
json.dump(test_results, f)
def train(model, train_dataloader, current_epoch, criterion, optimizer, writer):
batch_time = AverageMeter()
progbar = Progbar(len(train_dataloader.dataset), stateful_metrics=['epoch', 'config', 'lr'])
model.train()
end = time.time()
for step, (labels, imgs, img_paths) in enumerate(train_dataloader):
optimizer.zero_grad()
numm = imgs.shape[0]
imgs = imgs.reshape((-1, imgs.shape[-4], imgs.shape[-3], imgs.shape[-2], imgs.shape[-1])).permute((0, 2, 1, 3, 4))
imgs = Variable(imgs, requires_grad=True).cuda()
labels = Variable(labels, requires_grad=False).cuda().reshape(-1)
outputs = model(imgs).reshape(-1)
loss = criterion(outputs, labels.float())
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
progbar.add(numm, values=[('epoch', current_epoch),('train_loss', loss.item())])
writer.add_scalar('train_loss_epoch', loss.item(), current_epoch * len(train_dataloader.dataset) // train_dataloader.batch_size + step)
def begin_epoch(self, epoch, step, total_iters):
self.bar = Bar('{}'.format(self.exp_id), max=total_iters)
self.begin_step = step
self.total_iters = total_iters
self.total_iters_len = str(len(
str(total_iters))) # Used to format the log string
self.max_iters_len = str(len(str(
self.max_epochs * total_iters))) # Used to format the log string
self.bar_avg_loss = defaultdict(lambda: AverageMeter())
self.log_avg_loss = defaultdict(lambda: AverageMeter())
self.bar_batch_time = AverageMeter()
self.log_batch_time = AverageMeter()
self.start = time.time()
self.last_lrs = dict()
def validate(test_loader, model, args, config):
"""Test the model on the validation set
We follow "fully convolutional" testing:
* Scale the video with shortest side =256
* Uniformly sample 10 clips within a video
* For each clip, crop K=3 regions of 256*256 along the longest side
* This is equivalent to 30-crop testing
"""
# set up meters
batch_time = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
cm_meter = AverageMeter()
model.eval()
# data prefetcher with noramlization
test_loader = ClipPrefetcherJoint(test_loader, config['input']['mean'],
config['input']['std'])
# loop over validation set
end = time.time()
input, target = test_loader.next()
i = 0
# for large models
if args.slice:
batch_size = input.size(1)
max_split_size = 1
for split_size in range(2, batch_size):
if (batch_size % split_size) == 0 and split_size > max_split_size:
max_split_size = split_size
num_batch_splits = batch_size // max_split_size
print("Split the input by size: {:d}x{:d}".format(
max_split_size, num_batch_splits))
while input is not None:
i += 1
# disable/enable gradients
with torch.no_grad():
if args.slice:
# slice the inputs for testing
splited_inputs = torch.split(input, max_split_size, dim=1)
splited_outputs = []
for idx in range(num_batch_splits):
split_output = model(splited_inputs[idx])
# test time augmentation (minor performance boost)
flipped_split_input = torch.flip(splited_inputs[idx],
(-1, ))
flipped_split_output = model(flipped_split_input)
split_output = 0.5 * (split_output + flipped_split_output)
splited_outputs.append(split_output)
output = torch.mean(torch.stack(splited_outputs), dim=0)
else:
# forward all inputs
output, _, _ = model(input)
# print(output.size())
# test time augmentation (minor performance boost)
# always flip the last dim (width)
flipped_input = torch.flip(input, (-1, ))
flipped_output, _, _ = model(flipped_input)
output = 0.5 * (output + flipped_output)
# print(target[1].size())
# print(target[2].size())
# measure accuracy and record loss
acc1, acc5 = accuracy(output.data, target[0], topk=(1, 5))
top1.update(acc1.item(), input.size(0))
top5.update(acc5.item(), input.size(0))
batch_cm = confusion_matrix(output.data, target[0])
cm_meter.update(batch_cm.data.cpu().double())
# prefetch next batch
input, target = test_loader.next()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# printing
if i % (args.print_freq * 2) == 0:
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t'
'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format(
i,
len(test_loader),
batch_time=batch_time,
top1=top1,
top5=top5))
cls_acc = mean_class_accuracy(cm_meter.sum)
print(
'***Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f} Mean Cls Acc {cls_acc:.3f}'
.format(top1=top1, top5=top5, cls_acc=100 * cls_acc))
return top1.avg, top5.avg
class TrainCallback(object):
def __init__(self, opt, writer, logger):
self.writer = writer
self.logger = logger
self.exp_id = opt.run.exp_id
self.logging_n = opt.train.logging_n
self.max_epochs = opt.train.n_epochs
self.max_epochs_len = str(len(str(
self.max_epochs))) # Used to format the log string
def begin_epoch(self, epoch, step, total_iters):
self.bar = Bar('{}'.format(self.exp_id), max=total_iters)
self.begin_step = step
self.total_iters = total_iters
self.total_iters_len = str(len(
str(total_iters))) # Used to format the log string
self.max_iters_len = str(len(str(
self.max_epochs * total_iters))) # Used to format the log string
self.bar_avg_loss = defaultdict(lambda: AverageMeter())
self.log_avg_loss = defaultdict(lambda: AverageMeter())
self.bar_batch_time = AverageMeter()
self.log_batch_time = AverageMeter()
self.start = time.time()
self.last_lrs = dict()
def end_epoch(self, epoch, step):
self.bar.finish()
# Write learning rate(s) to TensorBoard
info = {('param/LR_' + l): v for l, v in self.last_lrs.items()}
for tag, value in info.items():
writer.add_scalar(tag, value, epoch)
def end_iter(self, epoch, step, losses, lrs, batch_size):
iter_id = step - self.begin_step
self.bar_batch_time.update(time.time() - self.start)
self.log_batch_time.update(time.time() - self.start)
# Update loss(es), learning rate(s) and timing values and display them in progressbar
Bar.suffix = '[{0}][{1}/{2}]|Tot: {total:} |ETA: {eta:} |Bch: {batch:.1f}s '.format(
epoch,
iter_id,
self.total_iters,
total=self.bar.elapsed_td,
eta=self.bar.eta_td,
batch=self.bar_batch_time.avg)
if len(losses) == 1:
self.bar_avg_loss['LOSS'].update(
list(losses.values())[0], batch_size)
self.log_avg_loss['LOSS'].update(
list(losses.values())[0], batch_size)
Bar.suffix += '|Loss: {:.4f} '.format(
self.bar_avg_loss['LOSS'].avg)
else:
Bar.suffix += '|Loss '
for l_name, l_val in losses.items():
self.bar_avg_loss[l_name].update(l_val, batch_size)
self.log_avg_loss[l_name].update(l_val, batch_size)
Bar.suffix += '{}: {:.4f} '.format(
l_name, self.bar_avg_loss[l_name].avg)
if len(lrs) == 1:
Bar.suffix += '|LR: {:.6f} '.format(list(lrs.values())[0])
else:
Bar.suffix += '|LR '
for lr_name, lr_val in lrs.items():
Bar.suffix += '{}: {:.6f} '.format(lr_name, lr_val)
self.last_lrs = lrs
self.bar.next()
# Log loss(es), learning rate(s) and timing values at specified interval
if step % self.logging_n == 0:
log_str = (
'Train [{0:0' + self.max_epochs_len + 'd},{1:0' +
self.max_iters_len + 'd}][{2:0' + self.total_iters_len +
'd}/{3}]|Total: {total:} |Batch: {batch:4.1f}s ').format(
epoch,
step,
iter_id,
self.total_iters,
total=self.bar.elapsed_td,
batch=self.log_batch_time.avg)
if len(losses) == 1:
log_str += '|Loss: {:.4f} '.format(
self.log_avg_loss['LOSS'].avg)
else:
log_str += '|Loss '
for l_name in losses.keys():
log_str += '{}: {:.4f} '.format(
l_name, self.log_avg_loss[l_name].avg)
if len(lrs) == 1:
log_str += '|LR: {:.6f} '.format(list(lrs.values())[0])
else:
log_str += '|LR '
for lr_name, lr_val in lrs.items():
log_str += '{}: {:.6f} '.format(lr_name, lr_val)
self.logger.info(log_str)
for log_avg in self.log_avg_loss.values():
log_avg.reset()
self.log_batch_time.reset()
# Write loss(es) to TensorBoard
info = {('loss/' + l): v for l, v in losses.items()}
for tag, value in info.items():
writer.add_scalar(tag, value, step)
self.start = time.time()
def train_adv_msk_epoch(epoch,
data_loaders,
model,
criterions,
optimizer,
opt,
epoch_logger,
batch_logger,
warm_up_epochs,
tb_writer=None):
print('train at epoch {}'.format(epoch))
model.train()
# pytroch version check
torch_version = float(torch.__version__[:3])
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
act_losses = AverageMeter()
msk_act_losses = AverageMeter()
place_losses = AverageMeter()
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses = AverageMeter()
act_accuracies = AverageMeter()
msk_act_accuracies = AverageMeter()
place_accuracies = AverageMeter()
data_loader_unmasked = data_loaders[0]
data_loader_masked = data_loaders[1]
end_time = time.time()
for i, ((inputs_unmasked, targets_unmasked, places_unmasked),
(inputs_masked, targets_masked, maskings)) in enumerate(
zip(data_loader_unmasked, data_loader_masked)):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets_unmasked = targets_unmasked.cuda()
places_unmasked = places_unmasked.cuda()
targets_masked = targets_masked.cuda()
# # ------------------------------------------------
# # Train using Action CE loss and Place ADV loss
# # ------------------------------------------------
if opt.model == 'vgg':
inputs_unmasked = inputs_unmasked.squeeze()
inputs_unmasked = Variable(inputs_unmasked)
targets_unmasked = Variable(targets_unmasked)
if opt.is_mask_entropy:
if opt.is_place_adv:
if opt.is_mask_adv:
outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model(
inputs_unmasked)
else:
outputs_unmasked, outputs_places_unmasked = model(
inputs_unmasked)
else:
if opt.is_mask_adv:
outputs_unmasked, outputs_rev_unmasked = model(
inputs_unmasked)
else:
outputs_unmasked = model(inputs_unmasked)
elif opt.is_mask_cross_entropy:
if opt.is_place_adv:
outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model(
inputs_unmasked)
else:
outputs_unmasked, outputs_rev_unmasked = model(inputs_unmasked)
loss_act = criterions['action_cross_entropy'](outputs_unmasked,
targets_unmasked)
if opt.is_place_adv:
loss_place = criterions['places_cross_entropy'](
outputs_places_unmasked, places_unmasked)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places_unmasked)
loss = loss_act + loss_place + opt.weight_entropy_loss * loss_place_entropy
else:
loss = loss_act + loss_place
else:
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places_unmasked)
loss = loss_act + opt.weight_entropy_loss * loss_place_entropy
else:
loss = loss_act
if torch_version < 0.4:
acc = calculate_accuracy(outputs_unmasked, targets_unmasked)
losses.update(loss.data[0], inputs_unmasked.size(0))
else:
act_acc = calculate_accuracy_pt_0_4(outputs_unmasked,
targets_unmasked)
if opt.is_place_adv:
if opt.is_place_soft:
_, places_hard_target = torch.max(places_unmasked, 1)
place_acc = calculate_accuracy_pt_0_4(
outputs_places_unmasked, places_hard_target)
else:
place_acc = calculate_accuracy_pt_0_4(
outputs_places_unmasked, places_unmasked)
place_losses.update(loss_place.item(), inputs_unmasked.size(0))
else:
place_losses.update(0, inputs_unmasked.size(0))
losses.update(loss.item(), inputs_unmasked.size(0))
act_losses.update(loss_act.item(), inputs_unmasked.size(0))
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses.update(loss_place_entropy.item(),
inputs_unmasked.size(0))
act_accuracies.update(act_acc, inputs_unmasked.size(0))
if opt.is_place_adv:
place_accuracies.update(place_acc, inputs_unmasked.size(0))
else:
place_accuracies.update(0, inputs_unmasked.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# -------------------------------------------------
# Train using Mask Action Entropy loss (maximize)
# -------------------------------------------------
print('num of actual masking_inds = {}/{}'.format(
torch.sum(maskings), maskings.shape[0]))
if opt.model == 'vgg':
inputs_masked = inputs_masked.squeeze()
inputs_masked = Variable(inputs_masked)
targets_masked = Variable(targets_masked)
if opt.is_mask_entropy:
if opt.is_place_adv:
if opt.is_mask_adv:
outputs_masked, outputs_places_masked, outputs_rev_masked = model(
inputs_masked)
else:
outputs_masked, outputs_places_masked = model(
inputs_masked)
else:
if opt.is_mask_adv:
outputs_masked, outputs_rev_masked = model(inputs_masked)
else:
outputs_masked = model(inputs_masked)
elif opt.is_mask_cross_entropy:
if opt.is_place_adv:
outputs_masked, outputs_places_masked, outputs_rev_masked = model(
inputs_masked)
else:
outputs_masked, outputs_rev_masked = model(inputs_masked)
if opt.is_mask_entropy:
if opt.is_mask_adv:
loss_action_entropy = criterions['mask_criterion'](
outputs_rev_masked)
else:
loss_action_entropy = criterions['mask_criterion'](
outputs_masked)
msk_loss = loss_action_entropy
elif opt.is_mask_cross_entropy:
loss_action_cross_entropy = criterions['mask_criterion'](
outputs_rev_masked, targets_masked)
msk_loss = loss_action_cross_entropy
if torch_version < 0.4:
if opt.is_mask_adv:
acc = calculate_accuracy(outputs_rev_masked, targets_masked)
else:
acc = calculate_accuracy(outputs_masked, targets_masked)
else:
if opt.is_mask_adv:
msk_act_acc = calculate_accuracy_pt_0_4(
outputs_rev_masked, targets_masked)
else:
msk_act_acc = calculate_accuracy_pt_0_4(
outputs_masked, targets_masked)
msk_act_losses.update(msk_loss.item(), inputs_masked.size(0))
msk_act_accuracies.update(msk_act_acc, inputs_masked.size(0))
optimizer.zero_grad()
msk_loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch':
epoch,
'batch':
i + 1,
'iter': (epoch - 1) * len(data_loader_unmasked) + (i + 1),
'loss total':
losses.val,
'loss act':
act_losses.val,
'loss place':
place_losses.val,
'acc act':
act_accuracies.val,
'acc place':
place_accuracies.val,
'lr':
optimizer.param_groups[0]['lr']
})
print(
'Train Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Total Loss {loss.val:.4f} ({loss.avg:.4f})\n'
'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t'
'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\t'
'Mask Action Confusion Loss {msk_loss.val:.4f} ({msk_loss.avg:.4f})\n'
'Acc Action {act_acc.val:.3f} ({act_acc.avg:.3f})\t'
'Acc Place {place_acc.val:.3f} ({place_acc.avg:.3f})\t'
'Acc Mask Action {msk_act_acc.val:.3f} ({msk_act_acc.avg:.3f})'.
format(epoch,
i + 1,
len(data_loader_unmasked),
batch_time=batch_time,
data_time=data_time,
loss=losses,
loss_act=act_losses,
loss_place=place_losses,
msk_loss=msk_act_losses,
act_acc=act_accuracies,
place_acc=place_accuracies,
msk_act_acc=msk_act_accuracies))
epoch_logger.log({
'epoch': epoch,
'loss total': losses.avg,
'loss act': act_losses.avg,
'loss place': place_losses.avg,
'acc act': act_accuracies.avg,
'acc place': place_accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
if tb_writer is not None:
tb_writer.add_scalar('train/loss/total', losses.avg, epoch)
tb_writer.add_scalar('train/loss/action', act_losses.avg, epoch)
tb_writer.add_scalar('train/loss/place', place_losses.avg, epoch)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
tb_writer.add_scalar('train/loss/place_entropy',
place_entropy_losses.avg, epoch)
tb_writer.add_scalar('train/acc/top1_acc_action', act_accuracies.avg,
epoch)
tb_writer.add_scalar('train/acc/top1_acc_place', place_accuracies.avg,
epoch)
if epoch < warm_up_epochs:
tb_writer.add_scalar('train/lr_new',
optimizer.param_groups[0]['lr'], epoch)
else:
tb_writer.add_scalar('train/lr_pretrained',
optimizer.param_groups[0]['lr'], epoch)
tb_writer.add_scalar('train/lr_new',
optimizer.param_groups[-1]['lr'], epoch)
tb_writer.add_scalar('train/msk_loss/msk_action', msk_act_losses.avg,
epoch)
tb_writer.add_scalar('train/msk_acc/top1_msk_acc_action',
msk_act_accuracies.avg, epoch)
def train_epoch(epoch,
data_loader,
model,
criterion,
optimizer,
opt,
epoch_logger,
batch_logger,
tb_writer=None):
print('train at epoch {}'.format(epoch))
model.train()
# pytroch version check
torch_version = float(torch.__version__[:3])
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda()
if opt.model == 'vgg':
inputs = inputs.squeeze()
inputs = Variable(inputs)
targets = Variable(targets)
outputs = model(inputs)
loss = criterion(outputs, targets)
if torch_version < 0.4:
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
else:
acc = calculate_accuracy_pt_0_4(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss': losses.val,
'acc': accuracies.val,
'lr': optimizer.param_groups[0]['lr']
})
print('Train Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
epoch_logger.log({
'epoch': epoch,
'loss': losses.avg,
'acc': accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
if tb_writer is not None:
tb_writer.add_scalar('train/loss/total', losses.avg, epoch)
tb_writer.add_scalar('train/acc/top1_acc_action', accuracies.avg,
epoch)
tb_writer.add_scalar('train/lr', optimizer.param_groups[0]['lr'],
epoch)
return batch_time.sum, data_time.sum
def val_epoch(epoch,
data_loader,
model,
criterion,
opt,
logger,
tb_writer=None):
print('validation at epoch {}'.format(epoch))
model.eval()
# pytroch version check
torch_version = float(torch.__version__[:3])
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, targets) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda()
if opt.model == 'vgg':
inputs = inputs.squeeze()
inputs = Variable(inputs, volatile=True)
targets = Variable(targets, volatile=True)
outputs = model(inputs)
loss = criterion(outputs, targets)
if torch_version < 0.4:
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
else:
acc = calculate_accuracy_pt_0_4(outputs, targets)
losses.update(loss.item(), inputs.size(0))
accuracies.update(acc, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Val Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
acc=accuracies))
logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg})
if tb_writer is not None:
tb_writer.add_scalar('val/loss/total', losses.avg, epoch)
tb_writer.add_scalar('val/acc/top1_acc_action', accuracies.avg, epoch)
return losses.avg, batch_time.sum, data_time.sum
def train(train_loader, model, criterion, optimizer, epoch, use_cuda):
# switch to train mode
model.train()
torch.set_grad_enabled(True)
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
end = time.time()
bar = Bar('Processing', max=len(train_loader))
show_step = len(train_loader) // 10
for batch_idx, (inputs, targets) in enumerate(train_loader):
batch_size = inputs.size(0)
if batch_size < args.train_batch:
continue
# measure data loading time
data_time.update(time.time() - end)
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda(async=True)
inputs, targets = torch.autograd.Variable(inputs), torch.autograd.Variable(targets)
# compute output
outputs = model(inputs)
loss = criterion(outputs, targets)
# measure accuracy and record loss
prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5))
losses.update(loss.data, inputs.size(0))
top1.update(prec1, inputs.size(0))
top5.update(prec5, inputs.size(0))
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.3f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format(
batch=batch_idx + 1,
size=len(train_loader),
data=data_time.val,
bt=batch_time.val,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
top1=top1.avg,
top5=top5.avg,
)
if (batch_idx) % show_step == 0:
print(bar.suffix)
bar.next()
bar.finish()
return (losses.avg, top1.avg)
def validate(val_loader, model, epoch, args, config):
"""Test the model on the validation set"""
# set up meters
batch_time = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
cm_meter = AverageMeter()
# switch to evaluate mode
model.eval()
# data prefetcher with noramlization
val_loader = ClipPrefetcherJoint(val_loader, config['input']['mean'],
config['input']['std'])
# loop over validation set
end = time.time()
input, target = val_loader.next()
i = 0
while input is not None:
i += 1
with torch.no_grad():
# forward the model (without gradients)
output = model(input)
# print(target[0])
# measure accuracy and record loss
acc1, acc5 = accuracy(output[0].data, target[0], topk=(1, 5))
batch_cm = confusion_matrix(output[0].data, target[0])
if args.distributed:
reduced_acc1 = reduce_tensor(acc1, args.world_size)
reduced_acc5 = reduce_tensor(acc5, args.world_size)
reduced_cm = reduce_tensor(batch_cm.data,
args.world_size,
avg=False)
else:
reduced_acc1 = acc1
reduced_acc5 = acc5
reduced_cm = batch_cm.data
top1.update(reduced_acc1.item(), input.size(0))
top5.update(reduced_acc5.item(), input.size(0))
cm_meter.update(reduced_cm.cpu().clone())
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# printing
if i % (args.print_freq * 2) == 0 and (args.local_rank == 0):
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t'
'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format(
i,
len(val_loader),
batch_time=batch_time,
top1=top1,
top5=top5))
# prefetch next batch
input, target = val_loader.next()
# finish up
if args.local_rank == 0:
cls_acc = 100 * mean_class_accuracy(cm_meter.sum)
print(
'******Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f} Cls Acc {cls_acc:.3f}'
.format(top1=top1, top5=top5, cls_acc=cls_acc))
# log top-1/5 acc
writer.add_scalars('data/top1_accuracy', {"val": top1.avg}, epoch + 1)
writer.add_scalars('data/top5_accuracy', {"val": top5.avg}, epoch + 1)
writer.add_scalars('data/mean_cls_acc', {"val": cls_acc}, epoch + 1)
return top1.avg, top5.avg
def train(train_loader, model, optimizer, scheduler, epoch, args, config):
"""Training the model"""
# set up meters
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
cm_meter = AverageMeter()
# number of iterations per epoch
num_iters = len(train_loader)
# switch to train mode
model.train()
# data prefetcher with noramlization
train_loader = ClipPrefetcherJoint(train_loader, config['input']['mean'],
config['input']['std'])
# main loop
end = time.time()
input, target = train_loader.next()
i = 0
while input is not None:
# input & target are pre-fetched
i += 1
# print(target)
# compute output
# print(input.size())
# print(target[0].size())
# print(target[1].size())
# print(target[2].size())
output, loss = model(input, targets=target)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
optimizer.step()
# printing (on the first GPU)
# print(i)
# print(args.print_freq)
if (i % args.print_freq) == 0:
# only check the stats when necessary
# avoid additional cost at each iter
acc1, acc5 = accuracy(output.data, target[0], topk=(1, 5))
batch_cm = confusion_matrix(output.data, target[0])
# measure accuracy and record loss
if args.distributed:
reduced_loss = reduce_tensor(loss.data, args.world_size)
reduced_acc1 = reduce_tensor(acc1, args.world_size)
reduced_acc5 = reduce_tensor(acc5, args.world_size)
reduced_cm = reduce_tensor(batch_cm.data,
args.world_size,
avg=False)
else:
reduced_loss = loss.mean().data
reduced_acc1 = acc1
reduced_acc5 = acc5
reduced_cm = batch_cm.data
losses.update(reduced_loss.item(), input.size(0))
top1.update(reduced_acc1.item(), input.size(0))
top5.update(reduced_acc5.item(), input.size(0))
cm_meter.update(reduced_cm.cpu().clone())
# measure elapsed time
torch.cuda.synchronize()
batch_time.update((time.time() - end) / args.print_freq)
end = time.time()
if args.local_rank == 0:
lr = scheduler.get_lr()[0]
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.3f} ({loss.avg:.3f})\t'
'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t'
'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format(
epoch + 1,
i,
num_iters,
batch_time=batch_time,
loss=losses,
top1=top1,
top5=top5))
# log loss / lr
writer.add_scalar('data/training_loss', losses.val,
epoch * num_iters + i)
writer.add_scalar('data/learning_rate', lr,
epoch * num_iters + i)
# step the lr scheduler after each iteration
scheduler.step()
# prefetch next batch
input, target = train_loader.next()
# finish up
if args.local_rank == 0:
# print & step the learning rate
lr = scheduler.get_lr()[0]
cls_acc = 100 * mean_class_accuracy(cm_meter.sum)
print("[Train]: Epoch {:d} finished with lr={:f}".format(
epoch + 1, lr))
# log top-1/5 acc
writer.add_scalars('data/top1_accuracy', {"train": top1.avg},
epoch + 1)
writer.add_scalars('data/top5_accuracy', {"train": top5.avg},
epoch + 1)
writer.add_scalars('data/mean_cls_acc', {"train": cls_acc}, epoch + 1)
return
input_std = 127.5
interpreter = tf.lite.Interpreter(model_path=float_model_file)
interpreter.allocate_tensors()
data_loader, data_loader_test = prepare_data_loaders(data_path)
# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
floating_model = input_details[0]['dtype'] == np.float32
height = input_details[0]['shape'][1]
width = input_details[0]['shape'][2]
top1 = AverageMeter('Acc@1', ':6.2f')
top5 = AverageMeter('Acc@5', ':6.2f')
cnt = 0
for image, target in data_loader_test:
image_resized = []
for im in image:
image_resized += [cv2.resize(im, (width, height))]
image = np.asarray(image_resized)
if floating_model:
input_data = (np.float32(image) - input_mean) / input_std
else:
input_data = image.astype(np.uint8)
interpreter.set_tensor(input_details[0]['index'], input_data)
def validate(test_loader, model, args, config):
"""Test the model on the validation set
We follow "fully convolutional" testing:
* Scale the video with shortest side =256
* Uniformly sample 10 clips within a video
* For each clip, crop K=3 regions of 256*256 along the longest side
* This is equivalent to 30-crop testing
"""
# set up meters
batch_time = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
cm_meter = AverageMeter()
# data prefetcher with noramlization
test_loader = ClipPrefetcherJointTest(test_loader, config['input']['mean'],
config['input']['std'])
# loop over validation set
end = time.time()
input, target, clipID = test_loader.next()
i = 0
mean = config['input']['mean']
std = config['input']['std']
mean_tensor = torch.tensor([255.0 * val
for val in mean]).cuda().view(1, 3, 1, 1, 1)
std_tensor = torch.tensor([255.0 * val
for val in std]).cuda().view(1, 3, 1, 1, 1)
# for large models
if args.slice:
batch_size = input.size(1)
max_split_size = 1
for split_size in range(2, batch_size):
if (batch_size % split_size) == 0 and split_size > max_split_size:
max_split_size = split_size
num_batch_splits = batch_size // max_split_size
print("Split the input by size: {:d}x{:d}".format(
max_split_size, num_batch_splits))
clipID_list = []
hand_gt_list = []
hotspot_gt_list = []
hand_pred_list = []
hotspot_pred_list = []
label_list = []
score_list = []
all_thresh = np.linspace(0.001, 1.0, 41)
tp = np.zeros((all_thresh.shape[0], ))
fp = np.zeros((all_thresh.shape[0], ))
fn = np.zeros((all_thresh.shape[0], ))
tn = np.zeros((all_thresh.shape[0], ))
count = 0
KL = 0
mean_error = 0
final_error = 0
while input is not None:
i += 1
# disable/enable gradients
with torch.no_grad():
if args.slice:
# slice the inputs for testing
splited_inputs = torch.split(input, max_split_size, dim=1)
splited_outputs = []
for idx in range(num_batch_splits):
split_output = model(splited_inputs[idx])
# test time augmentation (minor performance boost)
flipped_split_input = torch.flip(splited_inputs[idx],
(-1, ))
flipped_split_output = model(flipped_split_input)
split_output = 0.5 * (split_output + flipped_split_output)
splited_outputs.append(split_output)
output = torch.mean(torch.stack(splited_outputs), dim=0)
else:
# forward all inputs
# print(input.size())
output, hand_map, hotspot_map = model(input)
# test time augmentation (minor performance boost)
# always flip the last dim (width)
flipped_input = torch.flip(input, (-1, ))
flipped_output, _, _ = model(flipped_input)
output = 0.5 * (output + flipped_output)
input = input.mul(std_tensor).add_(mean_tensor)
instance_id = clipID[0].split('.')[0].split('_')[0]
print('./egtea_vis/' + instance_id + '.jpg')
# hand_map0 = np.squeeze(hand_map[0,:,:,:].cpu().numpy())
# hand_map1 = np.squeeze(hand_map[1,:,:,:].cpu().numpy())
# hand_map2 = np.squeeze(hand_map[2,:,:,:].cpu().numpy())
hand_map = np.squeeze(hand_map.cpu().numpy())
hotspot_map = np.squeeze(hotspot_map.cpu().numpy())
im = np.squeeze(input[0, :, -1, :, :].cpu().numpy())
im = im.transpose((1, 2, 0))
im = im[..., ::-1]
# print(hand_map0.shape)
# hand_map_i = hand_map[i,:,:,:]
# cv2.imwrite('./egtea_vis/'+instance_id+'.jpg',im)
hotspot_map_i = hotspot_map
hotspot_map_i = cv2.resize(hotspot_map_i, (256, 256),
interpolation=cv2.INTER_LINEAR)
hotspot_map_i = hotspot_map_i / np.max(hotspot_map_i) * 0.7
hotspot_map_i = (hotspot_map_i * 255).astype(np.uint8)
hotspot_map_i = cv2.applyColorMap(hotspot_map_i, cv2.COLORMAP_JET)
im = im.astype(np.uint8)
hotspot_map_i = hotspot_map_i.astype(np.uint8)
res = cv2.addWeighted(im, 0.5, hotspot_map_i, 0.5, 0)
cv2.imwrite('./egtea_vis/' + instance_id + 'hot.jpg',
hotspot_map_i)
# break
# assert np.max(res)<=255
# motor_atten=np.zeros((hand_map.shape[1],hand_map.shape[2]))
# for j in range(hand_map.shape[0]):
# motor_atten = motor_atten+hand_map[j,:,:]
# motor_atten = cv2.resize(motor_atten, (224, 224), interpolation=cv2.INTER_LINEAR)
# motor_atten = motor_atten/np.max(motor_atten)*0.9
# motor_atten = (motor_atten*255).astype(np.uint8)
# motor_atten = cv2.applyColorMap(motor_atten, cv2.COLORMAP_JET)
# res = im * 0.6 + motor_atten * 0.4
# cv2.imwrite('./handvis/'+instance_id+'motoratten.jpg',res)
# for j in range(hand_map.shape[0]):
# t = hand_map[j,:,:]
# t = cv2.resize(t, (224, 224), interpolation=cv2.INTER_LINEAR)
# pred_y,pred_x=np.unravel_index(t.argmax(), t.shape)
# if j == 0:
# im = cv2.circle(im, (pred_x,pred_y), 8, (0, 255, 255), -1)
# elif j==1:
# im = cv2.circle(im, (pred_x,pred_y), 8, (0, 255, 0), -1)
# elif j ==2:
# im = cv2.circle(im, (pred_x,pred_y), 8, (255, 255, 0), -1)
# else:
# im = cv2.circle(im, (pred_x,pred_y), 8, (255, 0, 255), -1)
# cv2.imwrite('./egtea_vis/'+instance_id+'hand.jpg',im)
# break
# label_list.append(target[0].cpu().numpy()[0])
# score_list.append(output.cpu().numpy())
# hand_gt = np.squeeze(target[1].cpu().numpy())
# # hotspot_gt = np.squeeze(target[2].cpu().numpy())
# # # print(hotspot_map)
# # hand_map = np.squeeze(hand_map.cpu().numpy())
# hand_map = np.squeeze(hand_map.cpu().numpy())
# hotspot_map = np.squeeze(hotspot_map.cpu().numpy())
# # break
# mean_error_i,final_error_i,empty = compare_hand(hand_gt, hand_map)
# # ctp, ctn, cfp, cfn,out,cropped = compare_hotspot(hotspot_gt, hotspot_map, all_thresh)
# # tp = tp + ctp
# # tn = tn + ctn
# # fp = fp + cfp
# # fn = fn + cfn
# if empty == False:
# count+=1
# # print(mean_error_i,final_error_i)
# mean_error+=mean_error_i
# final_error+=final_error_i
# print(tp,tn,fp,fn)
# print(output.cpu().numpy().shape)
# print(np.squeeze(hand_map.cpu().numpy()).shape)
# print(np.squeeze(target[1].cpu().numpy()).shape)
# print(np.squeeze(hotspot_map.cpu().numpy()).shape)
# print(np.squeeze(target[2].cpu().numpy()).shape)
# hand_pred_list.append(np.squeeze(hand_map.cpu().numpy()).tolist())
# hotspot_pred_list.append(np.squeeze(hotspot_map.cpu().numpy()).tolist())
# hand_gt_list.append(np.squeeze(target[1].cpu().numpy()).tolist())
# hotspot_gt_list.append(np.squeeze(target[2].cpu().numpy()).tolist())
# print(output.size())
# print(target[0].size())
# print(target[1].size())
# print(target[2].size())
# print(clipID[0])
# print(hand_map.size())
# print(hotspot_map.size())
# clipID_list.append(clipID[0])
# # output_list.append(np.squeeze(output.cpu().numpy()).tolist())
# # measure accuracy and record loss
# acc1, acc5 = accuracy(output.data, target[0], topk=(1, 5))
# top1.update(acc1.item(), input.size(0))
# top5.update(acc5.item(), input.size(0))
# batch_cm = confusion_matrix(output.data, target[0])
# cm_meter.update(batch_cm.data.cpu().double())
# # # prefetch next batch
# # # measure elapsed time
# batch_time.update(time.time() - end)
# end = time.time()
# # printing
# # if i % (args.print_freq * 2) == 0:
# # print('Test: [{0}/{1}]\t'
# # 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'.format(
# # i, len(test_loader), batch_time=batch_time))
# if i % (args.print_freq * 2) == 0:
# print('Test: [{0}/{1}]\t'
# 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
# 'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t'
# 'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format(
# i, len(test_loader), batch_time=batch_time,
# top1=top1, top5=top5))
input, target, clipID = test_loader.next()
def test(val_loader, model, criterion, epoch, use_cuda):
global best_acc
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
# torch.set_grad_enabled(False)
end = time.time()
if args.local_rank == 0:
bar = Bar('Processing', max=len(val_loader))
prefetcher = data_prefetcher(val_loader)
inputs, targets = prefetcher.next()
batch_idx = -1
while inputs is not None:
# for batch_idx, (inputs, targets) in enumerate(val_loader):
batch_idx += 1
# if use_cuda:
# inputs, targets = inputs.cuda(), targets.cuda()
# inputs, targets = torch.autograd.Variable(inputs, volatile=True), torch.autograd.Variable(targets)
# compute output
with torch.no_grad():
outputs = model(inputs)
loss = criterion(outputs, targets)
# measure accuracy and record loss
prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5))
reduced_loss = reduce_tensor(loss.data)
prec1 = reduce_tensor(prec1)
prec5 = reduce_tensor(prec5)
# to_python_float incurs a host<->device sync
losses.update(to_python_float(reduced_loss), inputs.size(0))
top1.update(to_python_float(prec1), inputs.size(0))
top5.update(to_python_float(prec5), inputs.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
if args.local_rank == 0:
bar.suffix = 'Valid({batch}/{size}) | Batch: {bt:.3f}s | Total: {total:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format(
batch=batch_idx + 1,
size=len(val_loader),
bt=batch_time.avg,
total=bar.elapsed_td,
loss=losses.avg,
top1=top1.avg,
top5=top5.avg,
)
bar.next()
inputs, targets = prefetcher.next()
if args.local_rank == 0:
print(bar.suffix)
bar.finish()
return (losses.avg, top1.avg)
def train(train_loader, model, criterion, optimizer, epoch, use_cuda):
# switch to train mode
model.train()
torch.set_grad_enabled(True)
batch_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
end = time.time()
if args.local_rank == 0:
bar = Bar('Processing', max=len(train_loader))
show_step = len(train_loader) // 10
prefetcher = data_prefetcher(train_loader)
inputs, targets = prefetcher.next()
batch_idx = -1
while inputs is not None:
# for batch_idx, (inputs, targets) in enumerate(train_loader):
batch_idx += 1
batch_size = inputs.size(0)
if batch_size < args.train_batch:
break
# measure data loading time
# if use_cuda:
# inputs, targets = inputs.cuda(), targets.cuda(async=True)
# inputs, targets = torch.autograd.Variable(inputs), torch.autograd.Variable(targets)
if args.mixup:
inputs, targets_a, targets_b, lam = mixup_data(inputs, targets, args.alpha, use_cuda)
outputs = model(inputs)
loss_func = mixup_criterion(targets_a, targets_b, lam)
old_loss = loss_func(criterion, outputs)
else:
outputs = model(inputs)
old_loss = criterion(outputs, targets)
# compute gradient and do SGD step
optimizer.zero_grad()
# loss.backward()
with amp.scale_loss(old_loss, optimizer) as loss:
loss.backward()
optimizer.step()
if batch_idx % args.print_freq == 0:
# measure accuracy and record loss
prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5))
reduced_loss = reduce_tensor(loss.data)
prec1 = reduce_tensor(prec1)
prec5 = reduce_tensor(prec5)
# to_python_float incurs a host<->device sync
losses.update(to_python_float(reduced_loss), inputs.size(0))
top1.update(to_python_float(prec1), inputs.size(0))
top5.update(to_python_float(prec5), inputs.size(0))
torch.cuda.synchronize()
# measure elapsed time
batch_time.update((time.time() - end) / args.print_freq)
end = time.time()
if args.local_rank == 0: # plot progress
bar.suffix = '({batch}/{size}) | Batch: {bt:.3f}s | Total: {total:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format(
batch=batch_idx + 1,
size=len(train_loader),
bt=batch_time.val,
total=bar.elapsed_td,
loss=losses.avg,
top1=top1.avg,
top5=top5.avg,
)
bar.next()
if (batch_idx) % show_step == 0 and args.local_rank == 0:
print('E%d' % (epoch) + bar.suffix)
inputs, targets = prefetcher.next()
if args.local_rank == 0:
bar.finish()
return (losses.avg, top1.avg)
def val_adv_msk_epoch(epoch,
data_loaders,
model,
criterions,
opt,
logger,
tb_writer=None):
print('validation at epoch {}'.format(epoch))
model.eval()
# pytroch version check
torch_version = float(torch.__version__[:3])
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
act_losses = AverageMeter()
msk_act_losses = AverageMeter()
place_losses = AverageMeter()
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses = AverageMeter()
act_accuracies = AverageMeter()
msk_act_accuracies = AverageMeter()
place_accuracies = AverageMeter()
data_loader_unmasked = data_loaders[0]
data_loader_masked = data_loaders[1]
end_time = time.time()
for i, ((inputs_unmasked, targets_unmasked, places_unmasked),
(inputs_masked, targets_masked, maskings)) in enumerate(
zip(data_loader_unmasked, data_loader_masked)):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets_unmasked = targets_unmasked.cuda()
places_unmasked = places_unmasked.cuda()
targets_masked = targets_masked.cuda()
# ----------------------------------------------------
# Validation on Action CE loss and Place ADV loss
# ----------------------------------------------------
if opt.model == 'vgg':
inputs_unmasked = inputs_unmasked.squeeze()
inputs_unmasked = Variable(inputs_unmasked, volatile=True)
targets_unmasked = Variable(targets_unmasked, volatile=True)
if opt.is_mask_entropy:
if opt.is_place_adv:
if opt.is_mask_adv:
outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model(
inputs_unmasked)
else:
outputs_unmasked, outputs_places_unmasked = model(
inputs_unmasked)
else:
if opt.is_mask_adv:
outputs_unmasked, outputs_rev_unmasked = model(
inputs_unmasked)
else:
outputs_unmasked = model(inputs_unmasked)
elif opt.is_mask_cross_entropy:
if opt.is_place_adv:
outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model(
inputs_unmasked)
else:
outputs_unmasked, outputs_rev_unmasked = model(inputs_unmasked)
loss_act = criterions['action_cross_entropy'](outputs_unmasked,
targets_unmasked)
if opt.is_place_adv:
loss_place = criterions['places_cross_entropy'](
outputs_places_unmasked, places_unmasked)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places_unmasked)
loss = loss_act + loss_place + loss_place_entropy
else:
loss = loss_act + loss_place
else:
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places_unmasked)
loss = loss_act + loss_place_entropy
else:
loss = loss_act
if torch_version < 0.4:
acc = calculate_accuracy(outputs_unmasked, targets_unmasked)
losses.update(loss.data[0], inputs_unmasked.size(0))
else:
act_acc = calculate_accuracy_pt_0_4(outputs_unmasked,
targets_unmasked)
if opt.is_place_adv:
if opt.is_place_soft:
_, places_hard_target = torch.max(places_unmasked, 1)
place_acc = calculate_accuracy_pt_0_4(
outputs_places_unmasked, places_hard_target)
else:
place_acc = calculate_accuracy_pt_0_4(
outputs_places_unmasked, places_unmasked)
place_losses.update(loss_place.item(), inputs_unmasked.size(0))
else:
place_losses.update(0, inputs_unmasked.size(0))
losses.update(loss.item(), inputs_unmasked.size(0))
act_losses.update(loss_act.item(), inputs_unmasked.size(0))
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses.update(loss_place_entropy.item(),
inputs_unmasked.size(0))
act_accuracies.update(act_acc, inputs_unmasked.size(0))
if opt.is_place_adv:
place_accuracies.update(place_acc, inputs_unmasked.size(0))
else:
place_accuracies.update(0, inputs_unmasked.size(0))
# ---------------------------------------------------
# Validation on Mask Action Entropy loss (maximize)
# ---------------------------------------------------
print('num of actual masking_inds = {}/{}'.format(
torch.sum(maskings), maskings.shape[0]))
if opt.model == 'vgg':
inputs_masked = inputs_masked.squeeze()
inputs_masked = Variable(inputs_masked, volatile=True)
targets_masked = Variable(targets_masked, volatile=True)
if opt.is_mask_entropy:
if opt.is_place_adv:
if opt.is_mask_adv:
outputs_masked, outputs_places_masked, outputs_rev_masked = model(
inputs_masked)
else:
outputs_masked, outputs_places_masked = model(
inputs_masked)
else:
if opt.is_mask_adv:
outputs_masked, outputs_rev_masked = model(inputs_masked)
else:
outputs_masked = model(inputs_masked)
elif opt.is_mask_cross_entropy:
if opt.is_place_adv:
outputs_masked, outputs_places_masked, outputs_rev_masked = model(
inputs_masked)
else:
outputs_masked, outputs_rev_masked = model(inputs_masked)
if opt.is_mask_entropy:
if opt.is_mask_adv:
loss_action_entropy = criterions['mask_criterion'](
outputs_rev_masked)
else:
loss_action_entropy = criterions['mask_criterion'](
outputs_masked)
msk_loss = loss_action_entropy
elif opt.is_mask_cross_entropy:
loss_action_cross_entropy = criterions['mask_criterion'](
outputs_masked, targets_masked)
msk_loss = loss_action_cross_entropy
if torch_version < 0.4:
if opt.is_mask_entropy:
if opt.is_mask_adv:
acc = calculate_accuracy(outputs_rev_masked,
targets_masked)
else:
acc = calculate_accuracy(outputs_masked, targets_masked)
elif opt.is_mask_cross_entropy:
acc = calculate_accuracy(outputs_rev_masked, targets_masked)
else:
if opt.is_mask_adv:
msk_act_acc = calculate_accuracy_pt_0_4(
outputs_rev_masked, targets_masked)
else:
msk_act_acc = calculate_accuracy_pt_0_4(
outputs_masked, targets_masked)
msk_act_losses.update(msk_loss.item(), inputs_masked.size(0))
msk_act_accuracies.update(msk_act_acc, inputs_masked.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print(
'Val Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Total Loss {loss.val:.4f} ({loss.avg:.4f})\n'
'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t'
'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\t'
'Mask Action Confusion Loss {msk_loss.val:.4f} ({msk_loss.avg:.4f})\n'
'Acc action {act_acc.val:.3f} ({act_acc.avg:.3f})\t'
'Acc place {place_acc.val:.3f} ({place_acc.avg:.3f})\t'
'Acc Mask Action {msk_act_acc.val:.3f} ({msk_act_acc.avg:.3f})'.
format(epoch,
i + 1,
len(data_loader_unmasked),
batch_time=batch_time,
data_time=data_time,
loss=losses,
loss_act=act_losses,
loss_place=place_losses,
msk_loss=msk_act_losses,
act_acc=act_accuracies,
place_acc=place_accuracies,
msk_act_acc=msk_act_accuracies))
logger.log({
'epoch': epoch,
'loss total': losses.avg,
'loss act': act_losses.avg,
'loss place': place_losses.avg,
'acc act': act_accuracies.avg,
'acc place': place_accuracies.avg
})
if tb_writer is not None:
tb_writer.add_scalar('val/loss/total', losses.avg, epoch)
tb_writer.add_scalar('val/loss/action', act_losses.avg, epoch)
tb_writer.add_scalar('val/loss/place', place_losses.avg, epoch)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
tb_writer.add_scalar('val/loss/place_entropy',
place_entropy_losses.avg, epoch)
tb_writer.add_scalar('val/acc/top1_acc_action', act_accuracies.avg,
epoch)
tb_writer.add_scalar('val/acc/top1_acc_place', place_accuracies.avg,
epoch)
tb_writer.add_scalar('val/msk_loss/msk_action', msk_act_losses.avg,
epoch)
tb_writer.add_scalar('val/msk_acc/top1_msk_acc_action',
msk_act_accuracies.avg, epoch)
return losses.avg
def main_baseline_model(opt):
torch.manual_seed(opt.manual_seed)
np.random.seed(opt.manual_seed)
model, parameters = generate_model(opt)
print(model)
criterion = nn.CrossEntropyLoss()
if not opt.no_cuda:
criterion = criterion.cuda()
if opt.no_mean_norm and not opt.std_norm:
norm_method = Normalize([0, 0, 0], [1, 1, 1])
elif not opt.std_norm:
norm_method = Normalize(opt.mean, [1, 1, 1])
else:
norm_method = Normalize(opt.mean, opt.std)
if not opt.no_train:
assert opt.train_crop in ['random', 'corner', 'center']
if opt.train_crop == 'random':
crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'corner':
crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size)
elif opt.train_crop == 'center':
crop_method = MultiScaleCornerCrop(opt.scales,
opt.sample_size,
crop_positions=['c'])
spatial_transform = Compose([
crop_method,
RandomHorizontalFlip(),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = TemporalRandomCrop(opt.sample_duration)
target_transform = ClassLabel()
training_data = get_training_set(opt, spatial_transform,
temporal_transform, target_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.n_threads,
pin_memory=True)
train_logger = Logger(os.path.join(opt.result_path, 'train.log'),
['epoch', 'loss', 'acc', 'lr'])
train_batch_logger = Logger(
os.path.join(opt.result_path, 'train_batch.log'),
['epoch', 'batch', 'iter', 'loss', 'acc', 'lr'])
if opt.nesterov:
dampening = 0
else:
dampening = opt.dampening
if opt.dataset in [
'kinetics_adv', 'kinetics_bkgmsk', 'kinetics_adv_msk'
]:
first_optimizer = optim.SGD(
parameters[1], # new parameters only
lr=opt.new_layer_lr,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
first_scheduler = lr_scheduler.ReduceLROnPlateau(
first_optimizer, 'min', patience=opt.lr_patience)
second_optimizer = optim.SGD(
[
{
'params': parameters[0]
}, # pretrained parameters
{
'params': parameters[1]
} # new parameters
],
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
second_scheduler = lr_scheduler.ReduceLROnPlateau(
second_optimizer, 'min', patience=opt.lr_patience)
else:
optimizer = optim.SGD(parameters,
lr=opt.learning_rate,
momentum=opt.momentum,
dampening=dampening,
weight_decay=opt.weight_decay,
nesterov=opt.nesterov)
scheduler = lr_scheduler.ReduceLROnPlateau(
optimizer, 'min', patience=opt.lr_patience)
if not opt.no_val:
spatial_transform = Compose([
Scale(opt.sample_size),
CenterCrop(opt.sample_size),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = ClassLabel()
validation_data = get_validation_set(opt, spatial_transform,
temporal_transform,
target_transform)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=opt.val_batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
val_logger = Logger(os.path.join(opt.result_path, 'val.log'),
['epoch', 'loss', 'acc'])
if opt.resume_path:
print('loading checkpoint {}'.format(opt.resume_path))
checkpoint = torch.load(opt.resume_path)
assert opt.arch == checkpoint['arch']
opt.begin_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
if not opt.no_train:
optimizer.load_state_dict(checkpoint['optimizer'])
print('run')
tr_btime_avg = AverageMeter()
tr_dtime_avg = AverageMeter()
val_btime_avg = AverageMeter()
val_dtime_avg = AverageMeter()
for i in range(opt.begin_epoch, opt.n_epochs + 1):
if opt.dataset in [
'kinetics_adv', 'kinetics_bkgmsk', 'kinetics_adv_msk'
]:
if i < opt.warm_up_epochs:
optimizer = first_optimizer
scheduler = first_scheduler
else:
optimizer = second_optimizer
scheduler = second_scheduler
if not opt.no_train:
tr_btime, tr_dtime = train_epoch(i, train_loader, model, criterion,
optimizer, opt, train_logger,
train_batch_logger)
if not opt.no_val:
validation_loss, val_btime, val_dtime = val_epoch(
i, val_loader, model, criterion, opt, val_logger)
if not opt.no_train and not opt.no_val:
scheduler.step(validation_loss)
tr_btime_avg.update(tr_btime)
tr_dtime_avg.update(tr_dtime)
val_btime_avg.update(val_btime)
val_dtime_avg.update(val_dtime)
print('One epoch tr btime = {:.2f}sec, tr dtime = {:.2f}'.format(
tr_btime_avg.avg, tr_dtime_avg.avg))
print('One epoch val btime = {:.2f}sec, val dtime = {:.2f}'.format(
val_btime_avg.avg, val_dtime_avg.avg))
sys.stdout.flush()
if opt.test:
spatial_transform = Compose([
Scale(int(opt.sample_size / opt.scale_in_test)),
CornerCrop(opt.sample_size, opt.crop_position_in_test),
ToTensor(opt.norm_value), norm_method
])
temporal_transform = LoopPadding(opt.sample_duration)
target_transform = VideoID()
test_data = get_test_set(opt, spatial_transform, temporal_transform,
target_transform)
test_loader = torch.utils.data.DataLoader(
test_data,
batch_size=opt.val_batch_size,
shuffle=False,
num_workers=opt.n_threads,
pin_memory=True)
test(test_loader, model, opt, test_data.class_names)
def val_adv_epoch(epoch,
data_loader,
model,
criterions,
opt,
logger,
tb_writer=None):
print('validation at epoch {}'.format(epoch))
model.eval()
# pytroch version check
torch_version = float(torch.__version__[:3])
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
act_losses = AverageMeter()
place_losses = AverageMeter()
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses = AverageMeter()
act_accuracies = AverageMeter()
place_accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, targets, places) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda()
places = places.cuda()
if opt.model == 'vgg':
inputs = inputs.squeeze()
inputs = Variable(inputs, volatile=True)
targets = Variable(targets, volatile=True)
if opt.is_place_adv:
outputs, outputs_places = model(inputs)
else:
outputs = model(inputs)
loss_act = criterions['action_cross_entropy'](outputs, targets)
if opt.is_place_adv:
loss_place = criterions['places_cross_entropy'](outputs_places,
places)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places)
loss = loss_act + loss_place + loss_place_entropy
else:
loss = loss_act + loss_place
else:
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places)
loss = loss_act + loss_place_entropy
else:
loss = loss_act
if torch_version < 0.4:
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
else:
act_acc = calculate_accuracy_pt_0_4(outputs, targets)
if opt.is_place_adv:
if opt.is_place_soft:
_, places_hard_target = torch.max(places, 1)
place_acc = calculate_accuracy_pt_0_4(
outputs_places, places_hard_target)
else:
place_acc = calculate_accuracy_pt_0_4(
outputs_places, places)
place_losses.update(loss_place.item(), inputs.size(0))
else:
place_losses.update(0, inputs.size(0))
losses.update(loss.item(), inputs.size(0))
act_losses.update(loss_act.item(), inputs.size(0))
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses.update(loss_place_entropy.item(),
inputs.size(0))
act_accuracies.update(act_acc, inputs.size(0))
if opt.is_place_adv:
place_accuracies.update(place_acc, inputs.size(0))
else:
place_accuracies.update(0, inputs.size(0))
batch_time.update(time.time() - end_time)
end_time = time.time()
print('Val Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\n'
'Total Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t'
'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\n'
'Acc action {act_acc.val:.3f} ({act_acc.avg:.3f})\t'
'Acc place {place_acc.val:.3f} ({place_acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
loss_act=act_losses,
loss_place=place_losses,
act_acc=act_accuracies,
place_acc=place_accuracies))
logger.log({
'epoch': epoch,
'loss total': losses.avg,
'loss act': act_losses.avg,
'loss place': place_losses.avg,
'acc act': act_accuracies.avg,
'acc place': place_accuracies.avg
})
if tb_writer is not None:
tb_writer.add_scalar('val/loss/total', losses.avg, epoch)
tb_writer.add_scalar('val/loss/action', act_losses.avg, epoch)
tb_writer.add_scalar('val/loss/place', place_losses.avg, epoch)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
tb_writer.add_scalar('val/loss/place_entropy',
place_entropy_losses.avg, epoch)
tb_writer.add_scalar('val/acc/top1_acc_action', act_accuracies.avg,
epoch)
tb_writer.add_scalar('val/acc/top1_acc_place', place_accuracies.avg,
epoch)
return losses.avg
def test(val_loader, model, criterion, epoch, use_cuda):
global best_acc
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
torch.set_grad_enabled(False)
end = time.time()
bar = Bar('Processing', max=len(val_loader))
for batch_idx, (inputs, targets) in enumerate(val_loader):
# measure data loading time
data_time.update(time.time() - end)
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = torch.autograd.Variable(inputs, volatile=True), torch.autograd.Variable(targets)
# compute output
outputs = model(inputs)
loss = criterion(outputs, targets)
# measure accuracy and record loss
prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5))
# losses.update(loss.data[0], inputs.size(0))
losses.update(loss.data, inputs.size(0))
#top1.update(prec1[0], inputs.size(0))
top1.update(prec1, inputs.size(0))
#top5.update(prec5[0], inputs.size(0))
top5.update(prec5, inputs.size(0))
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
# plot progress
bar.suffix = '({batch}/{size}) Data: {data:.3f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format(
batch=batch_idx + 1,
size=len(val_loader),
data=data_time.avg,
bt=batch_time.avg,
total=bar.elapsed_td,
eta=bar.eta_td,
loss=losses.avg,
top1=top1.avg,
top5=top5.avg,
)
bar.next()
print(bar.suffix)
bar.finish()
return (losses.avg, top1.avg)
def train_adv_epoch(epoch,
data_loader,
model,
criterions,
optimizer,
opt,
epoch_logger,
batch_logger,
warm_up_epochs,
tb_writer=None):
print('train at epoch {}'.format(epoch))
model.train()
# pytroch version check
torch_version = float(torch.__version__[:3])
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
act_losses = AverageMeter()
place_losses = AverageMeter()
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses = AverageMeter()
act_accuracies = AverageMeter()
place_accuracies = AverageMeter()
end_time = time.time()
for i, (inputs, targets, places) in enumerate(data_loader):
data_time.update(time.time() - end_time)
if not opt.no_cuda:
targets = targets.cuda(async=True)
places = places.cuda(async=True)
if opt.model == 'vgg':
inputs = inputs.squeeze()
inputs = Variable(inputs)
targets = Variable(targets)
if opt.is_place_adv:
outputs, outputs_places = model(inputs)
else:
outputs = model(inputs)
loss_act = criterions['action_cross_entropy'](outputs, targets)
if opt.is_place_adv:
loss_place = criterions['places_cross_entropy'](outputs_places,
places)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places)
loss = loss_act + loss_place + opt.weight_entropy_loss * loss_place_entropy
else:
loss = loss_act + loss_place
else:
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
loss_place_entropy = criterions['places_entropy'](
outputs_places)
loss = loss_act + opt.weight_entropy_loss * loss_place_entropy
else:
loss = loss_act
if torch_version < 0.4:
acc = calculate_accuracy(outputs, targets)
losses.update(loss.data[0], inputs.size(0))
else:
act_acc = calculate_accuracy_pt_0_4(outputs, targets)
if opt.is_place_adv:
if opt.is_place_soft:
_, places_hard_target = torch.max(places, 1)
place_acc = calculate_accuracy_pt_0_4(
outputs_places, places_hard_target)
else:
place_acc = calculate_accuracy_pt_0_4(
outputs_places, places)
place_losses.update(loss_place.item(), inputs.size(0))
else:
place_losses.update(0, inputs.size(0))
losses.update(loss.item(), inputs.size(0))
act_losses.update(loss_act.item(), inputs.size(0))
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
place_entropy_losses.update(loss_place_entropy.item(),
inputs.size(0))
act_accuracies.update(act_acc, inputs.size(0))
if opt.is_place_adv:
place_accuracies.update(place_acc, inputs.size(0))
else:
place_accuracies.update(0, inputs.size(0))
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_time.update(time.time() - end_time)
end_time = time.time()
batch_logger.log({
'epoch': epoch,
'batch': i + 1,
'iter': (epoch - 1) * len(data_loader) + (i + 1),
'loss total': losses.val,
'loss act': act_losses.val,
'loss place': place_losses.val,
'acc act': act_accuracies.val,
'acc place': place_accuracies.val,
'lr': optimizer.param_groups[0]['lr']
})
print('Train Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\n'
'Total Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t'
'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\n'
'Acc action {act_acc.val:.3f} ({act_acc.avg:.3f})\t'
'Acc place {place_acc.val:.3f} ({place_acc.avg:.3f})'.format(
epoch,
i + 1,
len(data_loader),
batch_time=batch_time,
data_time=data_time,
loss=losses,
loss_act=act_losses,
loss_place=place_losses,
act_acc=act_accuracies,
place_acc=place_accuracies))
epoch_logger.log({
'epoch': epoch,
'loss total': losses.avg,
'loss act': act_losses.avg,
'loss place': place_losses.avg,
'acc act': act_accuracies.avg,
'acc place': place_accuracies.avg,
'lr': optimizer.param_groups[0]['lr']
})
if epoch % opt.checkpoint == 0:
save_file_path = os.path.join(opt.result_path,
'save_{}.pth'.format(epoch))
states = {
'epoch': epoch + 1,
'arch': opt.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}
torch.save(states, save_file_path)
if tb_writer is not None:
tb_writer.add_scalar('train/loss/total', losses.avg, epoch)
tb_writer.add_scalar('train/loss/action', act_losses.avg, epoch)
tb_writer.add_scalar('train/loss/place', place_losses.avg, epoch)
if opt.is_place_entropy and epoch >= opt.warm_up_epochs:
tb_writer.add_scalar('train/loss/place_entropy',
place_entropy_losses.avg, epoch)
tb_writer.add_scalar('train/acc/top1_acc_action', act_accuracies.avg,
epoch)
tb_writer.add_scalar('train/acc/top1_acc_place', place_accuracies.avg,
epoch)
if epoch < warm_up_epochs:
tb_writer.add_scalar('train/lr_new',
optimizer.param_groups[0]['lr'], epoch)
else:
tb_writer.add_scalar('train/lr_pretrained',
optimizer.param_groups[0]['lr'], epoch)
tb_writer.add_scalar('train/lr_new',
optimizer.param_groups[-1]['lr'], epoch)