def validation(model, val_dataloader, current_epoch, criterion, writer, run_type='val',model_dir='./checkpoint'): batch_time = AverageMeter() losses = AverageMeter() model.eval() progbar = Progbar(len(val_dataloader.dataset), stateful_metrics=['run-type']) end = time.time() with torch.no_grad(): probs = np.array([]) gt_labels = np.array([]) names = [] for step, (labels, imgs, img_paths) in enumerate(val_dataloader): names.extend(img_paths) imgs = imgs.cuda() outputs = model(imgs).reshape(-1).cpu() loss = criterion(outputs,labels.float()) losses.update(loss) outputs = torch.sigmoid(outputs).cpu().numpy().reshape(-1) probs = np.concatenate((probs, outputs), axis=0) gt_labels = np.concatenate((gt_labels, labels.data.numpy().reshape(-1)), axis=0) assert gt_labels.shape == probs.shape progbar.add(imgs.size(0), values=[('run-type', run_type), ('val_loss', loss.item())]) # ,('batch_time', batch_time.val)]) batch_time.update(time.time() - end) end = time.time() metrixs = evaluate(gt_labels, probs > 0.5, probs) writer.add_scalar('val_loss_epoch', losses.avg, current_epoch) writer.add_scalar('val_f1',metrixs['f1']) print("Epoch: {} f1: {}".format(current_epoch, metrixs['f1'])) torch.save({ 'epoch': current_epoch + 1, 'state_dict': model.state_dict(), }, model_dir + '/model_last.pth.tar')
def prec_bn(train_loader, model, epoch, args, config): """precise batch norm""" # set up meters batch_time = AverageMeter() # number of iterations per epoch num_iters = len(train_loader) # switch to train mode & reduce batch norm momentum model.train() for m in model.modules(): if isinstance(m, nn.BatchNorm3d): m.momentum = 0.05 # data prefetcher with noramlization train_loader = ClipPrefetcherJoint(train_loader, config['input']['mean'], config['input']['std']) # main loop end = time.time() input, _ = train_loader.next() i = 0 while input is not None: # input & target are pre-fetched i += 1 # no gradient to params with torch.no_grad(): _ = model(input) # printing (on the first GPU) if (i % (args.print_freq * 2)) == 0 and (args.local_rank == 0): # measure elapsed time torch.cuda.synchronize() batch_time.update((time.time() - end) / (args.print_freq * 2)) end = time.time() print('Prec BN: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'.format( epoch + 1, i, num_iters, batch_time=batch_time)) # prefetch next batch input, _ = train_loader.next() # reset bn params back for m in model.modules(): if isinstance(m, nn.BatchNorm3d): m.momentum = 0.1 return
def test(data_loader, model, opt, class_names): print('test') model.eval() # pytroch version check torch_version = float(torch.__version__[:3]) batch_time = AverageMeter() data_time = AverageMeter() end_time = time.time() output_buffer = [] previous_video_id = '' test_results = {'results': {}} for i, (inputs, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) inputs = Variable(inputs, volatile=True) outputs = model(inputs) if not opt.no_softmax_in_test: outputs = F.softmax(outputs) for j in range(outputs.size(0)): if not (i == 0 and j == 0) and targets[j] != previous_video_id: if torch_version < 0.4: calculate_video_results(output_buffer, previous_video_id, test_results, class_names) else: calculate_video_results_pt_0_4(output_buffer, previous_video_id, test_results, class_names) output_buffer = [] output_buffer.append(outputs[j].data.cpu()) previous_video_id = targets[j] if (i % 100) == 0: with open( os.path.join(opt.result_path, '{}.json'.format(opt.test_subset)), 'w') as f: json.dump(test_results, f) batch_time.update(time.time() - end_time) end_time = time.time() print('[{}/{}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'.format( i + 1, len(data_loader), batch_time=batch_time, data_time=data_time)) with open(os.path.join(opt.result_path, '{}.json'.format(opt.test_subset)), 'w') as f: json.dump(test_results, f)
def train(model, train_dataloader, current_epoch, criterion, optimizer, writer): batch_time = AverageMeter() progbar = Progbar(len(train_dataloader.dataset), stateful_metrics=['epoch', 'config', 'lr']) model.train() end = time.time() for step, (labels, imgs, img_paths) in enumerate(train_dataloader): optimizer.zero_grad() numm = imgs.shape[0] imgs = imgs.reshape((-1, imgs.shape[-4], imgs.shape[-3], imgs.shape[-2], imgs.shape[-1])).permute((0, 2, 1, 3, 4)) imgs = Variable(imgs, requires_grad=True).cuda() labels = Variable(labels, requires_grad=False).cuda().reshape(-1) outputs = model(imgs).reshape(-1) loss = criterion(outputs, labels.float()) loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() progbar.add(numm, values=[('epoch', current_epoch),('train_loss', loss.item())]) writer.add_scalar('train_loss_epoch', loss.item(), current_epoch * len(train_dataloader.dataset) // train_dataloader.batch_size + step)
def begin_epoch(self, epoch, step, total_iters): self.bar = Bar('{}'.format(self.exp_id), max=total_iters) self.begin_step = step self.total_iters = total_iters self.total_iters_len = str(len( str(total_iters))) # Used to format the log string self.max_iters_len = str(len(str( self.max_epochs * total_iters))) # Used to format the log string self.bar_avg_loss = defaultdict(lambda: AverageMeter()) self.log_avg_loss = defaultdict(lambda: AverageMeter()) self.bar_batch_time = AverageMeter() self.log_batch_time = AverageMeter() self.start = time.time() self.last_lrs = dict()
def validate(test_loader, model, args, config): """Test the model on the validation set We follow "fully convolutional" testing: * Scale the video with shortest side =256 * Uniformly sample 10 clips within a video * For each clip, crop K=3 regions of 256*256 along the longest side * This is equivalent to 30-crop testing """ # set up meters batch_time = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() cm_meter = AverageMeter() model.eval() # data prefetcher with noramlization test_loader = ClipPrefetcherJoint(test_loader, config['input']['mean'], config['input']['std']) # loop over validation set end = time.time() input, target = test_loader.next() i = 0 # for large models if args.slice: batch_size = input.size(1) max_split_size = 1 for split_size in range(2, batch_size): if (batch_size % split_size) == 0 and split_size > max_split_size: max_split_size = split_size num_batch_splits = batch_size // max_split_size print("Split the input by size: {:d}x{:d}".format( max_split_size, num_batch_splits)) while input is not None: i += 1 # disable/enable gradients with torch.no_grad(): if args.slice: # slice the inputs for testing splited_inputs = torch.split(input, max_split_size, dim=1) splited_outputs = [] for idx in range(num_batch_splits): split_output = model(splited_inputs[idx]) # test time augmentation (minor performance boost) flipped_split_input = torch.flip(splited_inputs[idx], (-1, )) flipped_split_output = model(flipped_split_input) split_output = 0.5 * (split_output + flipped_split_output) splited_outputs.append(split_output) output = torch.mean(torch.stack(splited_outputs), dim=0) else: # forward all inputs output, _, _ = model(input) # print(output.size()) # test time augmentation (minor performance boost) # always flip the last dim (width) flipped_input = torch.flip(input, (-1, )) flipped_output, _, _ = model(flipped_input) output = 0.5 * (output + flipped_output) # print(target[1].size()) # print(target[2].size()) # measure accuracy and record loss acc1, acc5 = accuracy(output.data, target[0], topk=(1, 5)) top1.update(acc1.item(), input.size(0)) top5.update(acc5.item(), input.size(0)) batch_cm = confusion_matrix(output.data, target[0]) cm_meter.update(batch_cm.data.cpu().double()) # prefetch next batch input, target = test_loader.next() # measure elapsed time batch_time.update(time.time() - end) end = time.time() # printing if i % (args.print_freq * 2) == 0: print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t' 'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format( i, len(test_loader), batch_time=batch_time, top1=top1, top5=top5)) cls_acc = mean_class_accuracy(cm_meter.sum) print( '***Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f} Mean Cls Acc {cls_acc:.3f}' .format(top1=top1, top5=top5, cls_acc=100 * cls_acc)) return top1.avg, top5.avg
class TrainCallback(object): def __init__(self, opt, writer, logger): self.writer = writer self.logger = logger self.exp_id = opt.run.exp_id self.logging_n = opt.train.logging_n self.max_epochs = opt.train.n_epochs self.max_epochs_len = str(len(str( self.max_epochs))) # Used to format the log string def begin_epoch(self, epoch, step, total_iters): self.bar = Bar('{}'.format(self.exp_id), max=total_iters) self.begin_step = step self.total_iters = total_iters self.total_iters_len = str(len( str(total_iters))) # Used to format the log string self.max_iters_len = str(len(str( self.max_epochs * total_iters))) # Used to format the log string self.bar_avg_loss = defaultdict(lambda: AverageMeter()) self.log_avg_loss = defaultdict(lambda: AverageMeter()) self.bar_batch_time = AverageMeter() self.log_batch_time = AverageMeter() self.start = time.time() self.last_lrs = dict() def end_epoch(self, epoch, step): self.bar.finish() # Write learning rate(s) to TensorBoard info = {('param/LR_' + l): v for l, v in self.last_lrs.items()} for tag, value in info.items(): writer.add_scalar(tag, value, epoch) def end_iter(self, epoch, step, losses, lrs, batch_size): iter_id = step - self.begin_step self.bar_batch_time.update(time.time() - self.start) self.log_batch_time.update(time.time() - self.start) # Update loss(es), learning rate(s) and timing values and display them in progressbar Bar.suffix = '[{0}][{1}/{2}]|Tot: {total:} |ETA: {eta:} |Bch: {batch:.1f}s '.format( epoch, iter_id, self.total_iters, total=self.bar.elapsed_td, eta=self.bar.eta_td, batch=self.bar_batch_time.avg) if len(losses) == 1: self.bar_avg_loss['LOSS'].update( list(losses.values())[0], batch_size) self.log_avg_loss['LOSS'].update( list(losses.values())[0], batch_size) Bar.suffix += '|Loss: {:.4f} '.format( self.bar_avg_loss['LOSS'].avg) else: Bar.suffix += '|Loss ' for l_name, l_val in losses.items(): self.bar_avg_loss[l_name].update(l_val, batch_size) self.log_avg_loss[l_name].update(l_val, batch_size) Bar.suffix += '{}: {:.4f} '.format( l_name, self.bar_avg_loss[l_name].avg) if len(lrs) == 1: Bar.suffix += '|LR: {:.6f} '.format(list(lrs.values())[0]) else: Bar.suffix += '|LR ' for lr_name, lr_val in lrs.items(): Bar.suffix += '{}: {:.6f} '.format(lr_name, lr_val) self.last_lrs = lrs self.bar.next() # Log loss(es), learning rate(s) and timing values at specified interval if step % self.logging_n == 0: log_str = ( 'Train [{0:0' + self.max_epochs_len + 'd},{1:0' + self.max_iters_len + 'd}][{2:0' + self.total_iters_len + 'd}/{3}]|Total: {total:} |Batch: {batch:4.1f}s ').format( epoch, step, iter_id, self.total_iters, total=self.bar.elapsed_td, batch=self.log_batch_time.avg) if len(losses) == 1: log_str += '|Loss: {:.4f} '.format( self.log_avg_loss['LOSS'].avg) else: log_str += '|Loss ' for l_name in losses.keys(): log_str += '{}: {:.4f} '.format( l_name, self.log_avg_loss[l_name].avg) if len(lrs) == 1: log_str += '|LR: {:.6f} '.format(list(lrs.values())[0]) else: log_str += '|LR ' for lr_name, lr_val in lrs.items(): log_str += '{}: {:.6f} '.format(lr_name, lr_val) self.logger.info(log_str) for log_avg in self.log_avg_loss.values(): log_avg.reset() self.log_batch_time.reset() # Write loss(es) to TensorBoard info = {('loss/' + l): v for l, v in losses.items()} for tag, value in info.items(): writer.add_scalar(tag, value, step) self.start = time.time()
def train_adv_msk_epoch(epoch, data_loaders, model, criterions, optimizer, opt, epoch_logger, batch_logger, warm_up_epochs, tb_writer=None): print('train at epoch {}'.format(epoch)) model.train() # pytroch version check torch_version = float(torch.__version__[:3]) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() act_losses = AverageMeter() msk_act_losses = AverageMeter() place_losses = AverageMeter() if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses = AverageMeter() act_accuracies = AverageMeter() msk_act_accuracies = AverageMeter() place_accuracies = AverageMeter() data_loader_unmasked = data_loaders[0] data_loader_masked = data_loaders[1] end_time = time.time() for i, ((inputs_unmasked, targets_unmasked, places_unmasked), (inputs_masked, targets_masked, maskings)) in enumerate( zip(data_loader_unmasked, data_loader_masked)): data_time.update(time.time() - end_time) if not opt.no_cuda: targets_unmasked = targets_unmasked.cuda() places_unmasked = places_unmasked.cuda() targets_masked = targets_masked.cuda() # # ------------------------------------------------ # # Train using Action CE loss and Place ADV loss # # ------------------------------------------------ if opt.model == 'vgg': inputs_unmasked = inputs_unmasked.squeeze() inputs_unmasked = Variable(inputs_unmasked) targets_unmasked = Variable(targets_unmasked) if opt.is_mask_entropy: if opt.is_place_adv: if opt.is_mask_adv: outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model( inputs_unmasked) else: outputs_unmasked, outputs_places_unmasked = model( inputs_unmasked) else: if opt.is_mask_adv: outputs_unmasked, outputs_rev_unmasked = model( inputs_unmasked) else: outputs_unmasked = model(inputs_unmasked) elif opt.is_mask_cross_entropy: if opt.is_place_adv: outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model( inputs_unmasked) else: outputs_unmasked, outputs_rev_unmasked = model(inputs_unmasked) loss_act = criterions['action_cross_entropy'](outputs_unmasked, targets_unmasked) if opt.is_place_adv: loss_place = criterions['places_cross_entropy']( outputs_places_unmasked, places_unmasked) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places_unmasked) loss = loss_act + loss_place + opt.weight_entropy_loss * loss_place_entropy else: loss = loss_act + loss_place else: if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places_unmasked) loss = loss_act + opt.weight_entropy_loss * loss_place_entropy else: loss = loss_act if torch_version < 0.4: acc = calculate_accuracy(outputs_unmasked, targets_unmasked) losses.update(loss.data[0], inputs_unmasked.size(0)) else: act_acc = calculate_accuracy_pt_0_4(outputs_unmasked, targets_unmasked) if opt.is_place_adv: if opt.is_place_soft: _, places_hard_target = torch.max(places_unmasked, 1) place_acc = calculate_accuracy_pt_0_4( outputs_places_unmasked, places_hard_target) else: place_acc = calculate_accuracy_pt_0_4( outputs_places_unmasked, places_unmasked) place_losses.update(loss_place.item(), inputs_unmasked.size(0)) else: place_losses.update(0, inputs_unmasked.size(0)) losses.update(loss.item(), inputs_unmasked.size(0)) act_losses.update(loss_act.item(), inputs_unmasked.size(0)) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses.update(loss_place_entropy.item(), inputs_unmasked.size(0)) act_accuracies.update(act_acc, inputs_unmasked.size(0)) if opt.is_place_adv: place_accuracies.update(place_acc, inputs_unmasked.size(0)) else: place_accuracies.update(0, inputs_unmasked.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() # ------------------------------------------------- # Train using Mask Action Entropy loss (maximize) # ------------------------------------------------- print('num of actual masking_inds = {}/{}'.format( torch.sum(maskings), maskings.shape[0])) if opt.model == 'vgg': inputs_masked = inputs_masked.squeeze() inputs_masked = Variable(inputs_masked) targets_masked = Variable(targets_masked) if opt.is_mask_entropy: if opt.is_place_adv: if opt.is_mask_adv: outputs_masked, outputs_places_masked, outputs_rev_masked = model( inputs_masked) else: outputs_masked, outputs_places_masked = model( inputs_masked) else: if opt.is_mask_adv: outputs_masked, outputs_rev_masked = model(inputs_masked) else: outputs_masked = model(inputs_masked) elif opt.is_mask_cross_entropy: if opt.is_place_adv: outputs_masked, outputs_places_masked, outputs_rev_masked = model( inputs_masked) else: outputs_masked, outputs_rev_masked = model(inputs_masked) if opt.is_mask_entropy: if opt.is_mask_adv: loss_action_entropy = criterions['mask_criterion']( outputs_rev_masked) else: loss_action_entropy = criterions['mask_criterion']( outputs_masked) msk_loss = loss_action_entropy elif opt.is_mask_cross_entropy: loss_action_cross_entropy = criterions['mask_criterion']( outputs_rev_masked, targets_masked) msk_loss = loss_action_cross_entropy if torch_version < 0.4: if opt.is_mask_adv: acc = calculate_accuracy(outputs_rev_masked, targets_masked) else: acc = calculate_accuracy(outputs_masked, targets_masked) else: if opt.is_mask_adv: msk_act_acc = calculate_accuracy_pt_0_4( outputs_rev_masked, targets_masked) else: msk_act_acc = calculate_accuracy_pt_0_4( outputs_masked, targets_masked) msk_act_losses.update(msk_loss.item(), inputs_masked.size(0)) msk_act_accuracies.update(msk_act_acc, inputs_masked.size(0)) optimizer.zero_grad() msk_loss.backward() optimizer.step() batch_time.update(time.time() - end_time) end_time = time.time() batch_logger.log({ 'epoch': epoch, 'batch': i + 1, 'iter': (epoch - 1) * len(data_loader_unmasked) + (i + 1), 'loss total': losses.val, 'loss act': act_losses.val, 'loss place': place_losses.val, 'acc act': act_accuracies.val, 'acc place': place_accuracies.val, 'lr': optimizer.param_groups[0]['lr'] }) print( 'Train Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Total Loss {loss.val:.4f} ({loss.avg:.4f})\n' 'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t' 'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\t' 'Mask Action Confusion Loss {msk_loss.val:.4f} ({msk_loss.avg:.4f})\n' 'Acc Action {act_acc.val:.3f} ({act_acc.avg:.3f})\t' 'Acc Place {place_acc.val:.3f} ({place_acc.avg:.3f})\t' 'Acc Mask Action {msk_act_acc.val:.3f} ({msk_act_acc.avg:.3f})'. format(epoch, i + 1, len(data_loader_unmasked), batch_time=batch_time, data_time=data_time, loss=losses, loss_act=act_losses, loss_place=place_losses, msk_loss=msk_act_losses, act_acc=act_accuracies, place_acc=place_accuracies, msk_act_acc=msk_act_accuracies)) epoch_logger.log({ 'epoch': epoch, 'loss total': losses.avg, 'loss act': act_losses.avg, 'loss place': place_losses.avg, 'acc act': act_accuracies.avg, 'acc place': place_accuracies.avg, 'lr': optimizer.param_groups[0]['lr'] }) if epoch % opt.checkpoint == 0: save_file_path = os.path.join(opt.result_path, 'save_{}.pth'.format(epoch)) states = { 'epoch': epoch + 1, 'arch': opt.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), } torch.save(states, save_file_path) if tb_writer is not None: tb_writer.add_scalar('train/loss/total', losses.avg, epoch) tb_writer.add_scalar('train/loss/action', act_losses.avg, epoch) tb_writer.add_scalar('train/loss/place', place_losses.avg, epoch) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: tb_writer.add_scalar('train/loss/place_entropy', place_entropy_losses.avg, epoch) tb_writer.add_scalar('train/acc/top1_acc_action', act_accuracies.avg, epoch) tb_writer.add_scalar('train/acc/top1_acc_place', place_accuracies.avg, epoch) if epoch < warm_up_epochs: tb_writer.add_scalar('train/lr_new', optimizer.param_groups[0]['lr'], epoch) else: tb_writer.add_scalar('train/lr_pretrained', optimizer.param_groups[0]['lr'], epoch) tb_writer.add_scalar('train/lr_new', optimizer.param_groups[-1]['lr'], epoch) tb_writer.add_scalar('train/msk_loss/msk_action', msk_act_losses.avg, epoch) tb_writer.add_scalar('train/msk_acc/top1_msk_acc_action', msk_act_accuracies.avg, epoch)
def train_epoch(epoch, data_loader, model, criterion, optimizer, opt, epoch_logger, batch_logger, tb_writer=None): print('train at epoch {}'.format(epoch)) model.train() # pytroch version check torch_version = float(torch.__version__[:3]) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() accuracies = AverageMeter() end_time = time.time() for i, (inputs, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: targets = targets.cuda() if opt.model == 'vgg': inputs = inputs.squeeze() inputs = Variable(inputs) targets = Variable(targets) outputs = model(inputs) loss = criterion(outputs, targets) if torch_version < 0.4: acc = calculate_accuracy(outputs, targets) losses.update(loss.data[0], inputs.size(0)) else: acc = calculate_accuracy_pt_0_4(outputs, targets) losses.update(loss.item(), inputs.size(0)) accuracies.update(acc, inputs.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end_time) end_time = time.time() batch_logger.log({ 'epoch': epoch, 'batch': i + 1, 'iter': (epoch - 1) * len(data_loader) + (i + 1), 'loss': losses.val, 'acc': accuracies.val, 'lr': optimizer.param_groups[0]['lr'] }) print('Train Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch, i + 1, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, acc=accuracies)) epoch_logger.log({ 'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg, 'lr': optimizer.param_groups[0]['lr'] }) if epoch % opt.checkpoint == 0: save_file_path = os.path.join(opt.result_path, 'save_{}.pth'.format(epoch)) states = { 'epoch': epoch + 1, 'arch': opt.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), } torch.save(states, save_file_path) if tb_writer is not None: tb_writer.add_scalar('train/loss/total', losses.avg, epoch) tb_writer.add_scalar('train/acc/top1_acc_action', accuracies.avg, epoch) tb_writer.add_scalar('train/lr', optimizer.param_groups[0]['lr'], epoch) return batch_time.sum, data_time.sum
def val_epoch(epoch, data_loader, model, criterion, opt, logger, tb_writer=None): print('validation at epoch {}'.format(epoch)) model.eval() # pytroch version check torch_version = float(torch.__version__[:3]) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() accuracies = AverageMeter() end_time = time.time() for i, (inputs, targets) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: targets = targets.cuda() if opt.model == 'vgg': inputs = inputs.squeeze() inputs = Variable(inputs, volatile=True) targets = Variable(targets, volatile=True) outputs = model(inputs) loss = criterion(outputs, targets) if torch_version < 0.4: acc = calculate_accuracy(outputs, targets) losses.update(loss.data[0], inputs.size(0)) else: acc = calculate_accuracy_pt_0_4(outputs, targets) losses.update(loss.item(), inputs.size(0)) accuracies.update(acc, inputs.size(0)) batch_time.update(time.time() - end_time) end_time = time.time() print('Val Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc {acc.val:.3f} ({acc.avg:.3f})'.format(epoch, i + 1, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, acc=accuracies)) logger.log({'epoch': epoch, 'loss': losses.avg, 'acc': accuracies.avg}) if tb_writer is not None: tb_writer.add_scalar('val/loss/total', losses.avg, epoch) tb_writer.add_scalar('val/acc/top1_acc_action', accuracies.avg, epoch) return losses.avg, batch_time.sum, data_time.sum
def train(train_loader, model, criterion, optimizer, epoch, use_cuda): # switch to train mode model.train() torch.set_grad_enabled(True) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() end = time.time() bar = Bar('Processing', max=len(train_loader)) show_step = len(train_loader) // 10 for batch_idx, (inputs, targets) in enumerate(train_loader): batch_size = inputs.size(0) if batch_size < args.train_batch: continue # measure data loading time data_time.update(time.time() - end) if use_cuda: inputs, targets = inputs.cuda(), targets.cuda(async=True) inputs, targets = torch.autograd.Variable(inputs), torch.autograd.Variable(targets) # compute output outputs = model(inputs) loss = criterion(outputs, targets) # measure accuracy and record loss prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5)) losses.update(loss.data, inputs.size(0)) top1.update(prec1, inputs.size(0)) top5.update(prec5, inputs.size(0)) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # measure elapsed time batch_time.update(time.time() - end) end = time.time() # plot progress bar.suffix = '({batch}/{size}) Data: {data:.3f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format( batch=batch_idx + 1, size=len(train_loader), data=data_time.val, bt=batch_time.val, total=bar.elapsed_td, eta=bar.eta_td, loss=losses.avg, top1=top1.avg, top5=top5.avg, ) if (batch_idx) % show_step == 0: print(bar.suffix) bar.next() bar.finish() return (losses.avg, top1.avg)
def validate(val_loader, model, epoch, args, config): """Test the model on the validation set""" # set up meters batch_time = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() cm_meter = AverageMeter() # switch to evaluate mode model.eval() # data prefetcher with noramlization val_loader = ClipPrefetcherJoint(val_loader, config['input']['mean'], config['input']['std']) # loop over validation set end = time.time() input, target = val_loader.next() i = 0 while input is not None: i += 1 with torch.no_grad(): # forward the model (without gradients) output = model(input) # print(target[0]) # measure accuracy and record loss acc1, acc5 = accuracy(output[0].data, target[0], topk=(1, 5)) batch_cm = confusion_matrix(output[0].data, target[0]) if args.distributed: reduced_acc1 = reduce_tensor(acc1, args.world_size) reduced_acc5 = reduce_tensor(acc5, args.world_size) reduced_cm = reduce_tensor(batch_cm.data, args.world_size, avg=False) else: reduced_acc1 = acc1 reduced_acc5 = acc5 reduced_cm = batch_cm.data top1.update(reduced_acc1.item(), input.size(0)) top5.update(reduced_acc5.item(), input.size(0)) cm_meter.update(reduced_cm.cpu().clone()) # measure elapsed time batch_time.update(time.time() - end) end = time.time() # printing if i % (args.print_freq * 2) == 0 and (args.local_rank == 0): print('Test: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t' 'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format( i, len(val_loader), batch_time=batch_time, top1=top1, top5=top5)) # prefetch next batch input, target = val_loader.next() # finish up if args.local_rank == 0: cls_acc = 100 * mean_class_accuracy(cm_meter.sum) print( '******Acc@1 {top1.avg:.3f} Acc@5 {top5.avg:.3f} Cls Acc {cls_acc:.3f}' .format(top1=top1, top5=top5, cls_acc=cls_acc)) # log top-1/5 acc writer.add_scalars('data/top1_accuracy', {"val": top1.avg}, epoch + 1) writer.add_scalars('data/top5_accuracy', {"val": top5.avg}, epoch + 1) writer.add_scalars('data/mean_cls_acc', {"val": cls_acc}, epoch + 1) return top1.avg, top5.avg
def train(train_loader, model, optimizer, scheduler, epoch, args, config): """Training the model""" # set up meters batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() cm_meter = AverageMeter() # number of iterations per epoch num_iters = len(train_loader) # switch to train mode model.train() # data prefetcher with noramlization train_loader = ClipPrefetcherJoint(train_loader, config['input']['mean'], config['input']['std']) # main loop end = time.time() input, target = train_loader.next() i = 0 while input is not None: # input & target are pre-fetched i += 1 # print(target) # compute output # print(input.size()) # print(target[0].size()) # print(target[1].size()) # print(target[2].size()) output, loss = model(input, targets=target) # compute gradient and do SGD step optimizer.zero_grad() loss.backward() optimizer.step() # printing (on the first GPU) # print(i) # print(args.print_freq) if (i % args.print_freq) == 0: # only check the stats when necessary # avoid additional cost at each iter acc1, acc5 = accuracy(output.data, target[0], topk=(1, 5)) batch_cm = confusion_matrix(output.data, target[0]) # measure accuracy and record loss if args.distributed: reduced_loss = reduce_tensor(loss.data, args.world_size) reduced_acc1 = reduce_tensor(acc1, args.world_size) reduced_acc5 = reduce_tensor(acc5, args.world_size) reduced_cm = reduce_tensor(batch_cm.data, args.world_size, avg=False) else: reduced_loss = loss.mean().data reduced_acc1 = acc1 reduced_acc5 = acc5 reduced_cm = batch_cm.data losses.update(reduced_loss.item(), input.size(0)) top1.update(reduced_acc1.item(), input.size(0)) top5.update(reduced_acc5.item(), input.size(0)) cm_meter.update(reduced_cm.cpu().clone()) # measure elapsed time torch.cuda.synchronize() batch_time.update((time.time() - end) / args.print_freq) end = time.time() if args.local_rank == 0: lr = scheduler.get_lr()[0] print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.3f} ({loss.avg:.3f})\t' 'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t' 'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format( epoch + 1, i, num_iters, batch_time=batch_time, loss=losses, top1=top1, top5=top5)) # log loss / lr writer.add_scalar('data/training_loss', losses.val, epoch * num_iters + i) writer.add_scalar('data/learning_rate', lr, epoch * num_iters + i) # step the lr scheduler after each iteration scheduler.step() # prefetch next batch input, target = train_loader.next() # finish up if args.local_rank == 0: # print & step the learning rate lr = scheduler.get_lr()[0] cls_acc = 100 * mean_class_accuracy(cm_meter.sum) print("[Train]: Epoch {:d} finished with lr={:f}".format( epoch + 1, lr)) # log top-1/5 acc writer.add_scalars('data/top1_accuracy', {"train": top1.avg}, epoch + 1) writer.add_scalars('data/top5_accuracy', {"train": top5.avg}, epoch + 1) writer.add_scalars('data/mean_cls_acc', {"train": cls_acc}, epoch + 1) return
input_std = 127.5 interpreter = tf.lite.Interpreter(model_path=float_model_file) interpreter.allocate_tensors() data_loader, data_loader_test = prepare_data_loaders(data_path) # Get input and output tensors. input_details = interpreter.get_input_details() output_details = interpreter.get_output_details() floating_model = input_details[0]['dtype'] == np.float32 height = input_details[0]['shape'][1] width = input_details[0]['shape'][2] top1 = AverageMeter('Acc@1', ':6.2f') top5 = AverageMeter('Acc@5', ':6.2f') cnt = 0 for image, target in data_loader_test: image_resized = [] for im in image: image_resized += [cv2.resize(im, (width, height))] image = np.asarray(image_resized) if floating_model: input_data = (np.float32(image) - input_mean) / input_std else: input_data = image.astype(np.uint8) interpreter.set_tensor(input_details[0]['index'], input_data)
def validate(test_loader, model, args, config): """Test the model on the validation set We follow "fully convolutional" testing: * Scale the video with shortest side =256 * Uniformly sample 10 clips within a video * For each clip, crop K=3 regions of 256*256 along the longest side * This is equivalent to 30-crop testing """ # set up meters batch_time = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() cm_meter = AverageMeter() # data prefetcher with noramlization test_loader = ClipPrefetcherJointTest(test_loader, config['input']['mean'], config['input']['std']) # loop over validation set end = time.time() input, target, clipID = test_loader.next() i = 0 mean = config['input']['mean'] std = config['input']['std'] mean_tensor = torch.tensor([255.0 * val for val in mean]).cuda().view(1, 3, 1, 1, 1) std_tensor = torch.tensor([255.0 * val for val in std]).cuda().view(1, 3, 1, 1, 1) # for large models if args.slice: batch_size = input.size(1) max_split_size = 1 for split_size in range(2, batch_size): if (batch_size % split_size) == 0 and split_size > max_split_size: max_split_size = split_size num_batch_splits = batch_size // max_split_size print("Split the input by size: {:d}x{:d}".format( max_split_size, num_batch_splits)) clipID_list = [] hand_gt_list = [] hotspot_gt_list = [] hand_pred_list = [] hotspot_pred_list = [] label_list = [] score_list = [] all_thresh = np.linspace(0.001, 1.0, 41) tp = np.zeros((all_thresh.shape[0], )) fp = np.zeros((all_thresh.shape[0], )) fn = np.zeros((all_thresh.shape[0], )) tn = np.zeros((all_thresh.shape[0], )) count = 0 KL = 0 mean_error = 0 final_error = 0 while input is not None: i += 1 # disable/enable gradients with torch.no_grad(): if args.slice: # slice the inputs for testing splited_inputs = torch.split(input, max_split_size, dim=1) splited_outputs = [] for idx in range(num_batch_splits): split_output = model(splited_inputs[idx]) # test time augmentation (minor performance boost) flipped_split_input = torch.flip(splited_inputs[idx], (-1, )) flipped_split_output = model(flipped_split_input) split_output = 0.5 * (split_output + flipped_split_output) splited_outputs.append(split_output) output = torch.mean(torch.stack(splited_outputs), dim=0) else: # forward all inputs # print(input.size()) output, hand_map, hotspot_map = model(input) # test time augmentation (minor performance boost) # always flip the last dim (width) flipped_input = torch.flip(input, (-1, )) flipped_output, _, _ = model(flipped_input) output = 0.5 * (output + flipped_output) input = input.mul(std_tensor).add_(mean_tensor) instance_id = clipID[0].split('.')[0].split('_')[0] print('./egtea_vis/' + instance_id + '.jpg') # hand_map0 = np.squeeze(hand_map[0,:,:,:].cpu().numpy()) # hand_map1 = np.squeeze(hand_map[1,:,:,:].cpu().numpy()) # hand_map2 = np.squeeze(hand_map[2,:,:,:].cpu().numpy()) hand_map = np.squeeze(hand_map.cpu().numpy()) hotspot_map = np.squeeze(hotspot_map.cpu().numpy()) im = np.squeeze(input[0, :, -1, :, :].cpu().numpy()) im = im.transpose((1, 2, 0)) im = im[..., ::-1] # print(hand_map0.shape) # hand_map_i = hand_map[i,:,:,:] # cv2.imwrite('./egtea_vis/'+instance_id+'.jpg',im) hotspot_map_i = hotspot_map hotspot_map_i = cv2.resize(hotspot_map_i, (256, 256), interpolation=cv2.INTER_LINEAR) hotspot_map_i = hotspot_map_i / np.max(hotspot_map_i) * 0.7 hotspot_map_i = (hotspot_map_i * 255).astype(np.uint8) hotspot_map_i = cv2.applyColorMap(hotspot_map_i, cv2.COLORMAP_JET) im = im.astype(np.uint8) hotspot_map_i = hotspot_map_i.astype(np.uint8) res = cv2.addWeighted(im, 0.5, hotspot_map_i, 0.5, 0) cv2.imwrite('./egtea_vis/' + instance_id + 'hot.jpg', hotspot_map_i) # break # assert np.max(res)<=255 # motor_atten=np.zeros((hand_map.shape[1],hand_map.shape[2])) # for j in range(hand_map.shape[0]): # motor_atten = motor_atten+hand_map[j,:,:] # motor_atten = cv2.resize(motor_atten, (224, 224), interpolation=cv2.INTER_LINEAR) # motor_atten = motor_atten/np.max(motor_atten)*0.9 # motor_atten = (motor_atten*255).astype(np.uint8) # motor_atten = cv2.applyColorMap(motor_atten, cv2.COLORMAP_JET) # res = im * 0.6 + motor_atten * 0.4 # cv2.imwrite('./handvis/'+instance_id+'motoratten.jpg',res) # for j in range(hand_map.shape[0]): # t = hand_map[j,:,:] # t = cv2.resize(t, (224, 224), interpolation=cv2.INTER_LINEAR) # pred_y,pred_x=np.unravel_index(t.argmax(), t.shape) # if j == 0: # im = cv2.circle(im, (pred_x,pred_y), 8, (0, 255, 255), -1) # elif j==1: # im = cv2.circle(im, (pred_x,pred_y), 8, (0, 255, 0), -1) # elif j ==2: # im = cv2.circle(im, (pred_x,pred_y), 8, (255, 255, 0), -1) # else: # im = cv2.circle(im, (pred_x,pred_y), 8, (255, 0, 255), -1) # cv2.imwrite('./egtea_vis/'+instance_id+'hand.jpg',im) # break # label_list.append(target[0].cpu().numpy()[0]) # score_list.append(output.cpu().numpy()) # hand_gt = np.squeeze(target[1].cpu().numpy()) # # hotspot_gt = np.squeeze(target[2].cpu().numpy()) # # # print(hotspot_map) # # hand_map = np.squeeze(hand_map.cpu().numpy()) # hand_map = np.squeeze(hand_map.cpu().numpy()) # hotspot_map = np.squeeze(hotspot_map.cpu().numpy()) # # break # mean_error_i,final_error_i,empty = compare_hand(hand_gt, hand_map) # # ctp, ctn, cfp, cfn,out,cropped = compare_hotspot(hotspot_gt, hotspot_map, all_thresh) # # tp = tp + ctp # # tn = tn + ctn # # fp = fp + cfp # # fn = fn + cfn # if empty == False: # count+=1 # # print(mean_error_i,final_error_i) # mean_error+=mean_error_i # final_error+=final_error_i # print(tp,tn,fp,fn) # print(output.cpu().numpy().shape) # print(np.squeeze(hand_map.cpu().numpy()).shape) # print(np.squeeze(target[1].cpu().numpy()).shape) # print(np.squeeze(hotspot_map.cpu().numpy()).shape) # print(np.squeeze(target[2].cpu().numpy()).shape) # hand_pred_list.append(np.squeeze(hand_map.cpu().numpy()).tolist()) # hotspot_pred_list.append(np.squeeze(hotspot_map.cpu().numpy()).tolist()) # hand_gt_list.append(np.squeeze(target[1].cpu().numpy()).tolist()) # hotspot_gt_list.append(np.squeeze(target[2].cpu().numpy()).tolist()) # print(output.size()) # print(target[0].size()) # print(target[1].size()) # print(target[2].size()) # print(clipID[0]) # print(hand_map.size()) # print(hotspot_map.size()) # clipID_list.append(clipID[0]) # # output_list.append(np.squeeze(output.cpu().numpy()).tolist()) # # measure accuracy and record loss # acc1, acc5 = accuracy(output.data, target[0], topk=(1, 5)) # top1.update(acc1.item(), input.size(0)) # top5.update(acc5.item(), input.size(0)) # batch_cm = confusion_matrix(output.data, target[0]) # cm_meter.update(batch_cm.data.cpu().double()) # # # prefetch next batch # # # measure elapsed time # batch_time.update(time.time() - end) # end = time.time() # # printing # # if i % (args.print_freq * 2) == 0: # # print('Test: [{0}/{1}]\t' # # 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'.format( # # i, len(test_loader), batch_time=batch_time)) # if i % (args.print_freq * 2) == 0: # print('Test: [{0}/{1}]\t' # 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' # 'Acc@1 {top1.val:.2f} ({top1.avg:.2f})\t' # 'Acc@5 {top5.val:.2f} ({top5.avg:.2f})'.format( # i, len(test_loader), batch_time=batch_time, # top1=top1, top5=top5)) input, target, clipID = test_loader.next()
def test(val_loader, model, criterion, epoch, use_cuda): global best_acc batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() # torch.set_grad_enabled(False) end = time.time() if args.local_rank == 0: bar = Bar('Processing', max=len(val_loader)) prefetcher = data_prefetcher(val_loader) inputs, targets = prefetcher.next() batch_idx = -1 while inputs is not None: # for batch_idx, (inputs, targets) in enumerate(val_loader): batch_idx += 1 # if use_cuda: # inputs, targets = inputs.cuda(), targets.cuda() # inputs, targets = torch.autograd.Variable(inputs, volatile=True), torch.autograd.Variable(targets) # compute output with torch.no_grad(): outputs = model(inputs) loss = criterion(outputs, targets) # measure accuracy and record loss prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5)) reduced_loss = reduce_tensor(loss.data) prec1 = reduce_tensor(prec1) prec5 = reduce_tensor(prec5) # to_python_float incurs a host<->device sync losses.update(to_python_float(reduced_loss), inputs.size(0)) top1.update(to_python_float(prec1), inputs.size(0)) top5.update(to_python_float(prec5), inputs.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() # plot progress if args.local_rank == 0: bar.suffix = 'Valid({batch}/{size}) | Batch: {bt:.3f}s | Total: {total:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format( batch=batch_idx + 1, size=len(val_loader), bt=batch_time.avg, total=bar.elapsed_td, loss=losses.avg, top1=top1.avg, top5=top5.avg, ) bar.next() inputs, targets = prefetcher.next() if args.local_rank == 0: print(bar.suffix) bar.finish() return (losses.avg, top1.avg)
def train(train_loader, model, criterion, optimizer, epoch, use_cuda): # switch to train mode model.train() torch.set_grad_enabled(True) batch_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() end = time.time() if args.local_rank == 0: bar = Bar('Processing', max=len(train_loader)) show_step = len(train_loader) // 10 prefetcher = data_prefetcher(train_loader) inputs, targets = prefetcher.next() batch_idx = -1 while inputs is not None: # for batch_idx, (inputs, targets) in enumerate(train_loader): batch_idx += 1 batch_size = inputs.size(0) if batch_size < args.train_batch: break # measure data loading time # if use_cuda: # inputs, targets = inputs.cuda(), targets.cuda(async=True) # inputs, targets = torch.autograd.Variable(inputs), torch.autograd.Variable(targets) if args.mixup: inputs, targets_a, targets_b, lam = mixup_data(inputs, targets, args.alpha, use_cuda) outputs = model(inputs) loss_func = mixup_criterion(targets_a, targets_b, lam) old_loss = loss_func(criterion, outputs) else: outputs = model(inputs) old_loss = criterion(outputs, targets) # compute gradient and do SGD step optimizer.zero_grad() # loss.backward() with amp.scale_loss(old_loss, optimizer) as loss: loss.backward() optimizer.step() if batch_idx % args.print_freq == 0: # measure accuracy and record loss prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5)) reduced_loss = reduce_tensor(loss.data) prec1 = reduce_tensor(prec1) prec5 = reduce_tensor(prec5) # to_python_float incurs a host<->device sync losses.update(to_python_float(reduced_loss), inputs.size(0)) top1.update(to_python_float(prec1), inputs.size(0)) top5.update(to_python_float(prec5), inputs.size(0)) torch.cuda.synchronize() # measure elapsed time batch_time.update((time.time() - end) / args.print_freq) end = time.time() if args.local_rank == 0: # plot progress bar.suffix = '({batch}/{size}) | Batch: {bt:.3f}s | Total: {total:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format( batch=batch_idx + 1, size=len(train_loader), bt=batch_time.val, total=bar.elapsed_td, loss=losses.avg, top1=top1.avg, top5=top5.avg, ) bar.next() if (batch_idx) % show_step == 0 and args.local_rank == 0: print('E%d' % (epoch) + bar.suffix) inputs, targets = prefetcher.next() if args.local_rank == 0: bar.finish() return (losses.avg, top1.avg)
def val_adv_msk_epoch(epoch, data_loaders, model, criterions, opt, logger, tb_writer=None): print('validation at epoch {}'.format(epoch)) model.eval() # pytroch version check torch_version = float(torch.__version__[:3]) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() act_losses = AverageMeter() msk_act_losses = AverageMeter() place_losses = AverageMeter() if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses = AverageMeter() act_accuracies = AverageMeter() msk_act_accuracies = AverageMeter() place_accuracies = AverageMeter() data_loader_unmasked = data_loaders[0] data_loader_masked = data_loaders[1] end_time = time.time() for i, ((inputs_unmasked, targets_unmasked, places_unmasked), (inputs_masked, targets_masked, maskings)) in enumerate( zip(data_loader_unmasked, data_loader_masked)): data_time.update(time.time() - end_time) if not opt.no_cuda: targets_unmasked = targets_unmasked.cuda() places_unmasked = places_unmasked.cuda() targets_masked = targets_masked.cuda() # ---------------------------------------------------- # Validation on Action CE loss and Place ADV loss # ---------------------------------------------------- if opt.model == 'vgg': inputs_unmasked = inputs_unmasked.squeeze() inputs_unmasked = Variable(inputs_unmasked, volatile=True) targets_unmasked = Variable(targets_unmasked, volatile=True) if opt.is_mask_entropy: if opt.is_place_adv: if opt.is_mask_adv: outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model( inputs_unmasked) else: outputs_unmasked, outputs_places_unmasked = model( inputs_unmasked) else: if opt.is_mask_adv: outputs_unmasked, outputs_rev_unmasked = model( inputs_unmasked) else: outputs_unmasked = model(inputs_unmasked) elif opt.is_mask_cross_entropy: if opt.is_place_adv: outputs_unmasked, outputs_places_unmasked, outputs_rev_unmasked = model( inputs_unmasked) else: outputs_unmasked, outputs_rev_unmasked = model(inputs_unmasked) loss_act = criterions['action_cross_entropy'](outputs_unmasked, targets_unmasked) if opt.is_place_adv: loss_place = criterions['places_cross_entropy']( outputs_places_unmasked, places_unmasked) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places_unmasked) loss = loss_act + loss_place + loss_place_entropy else: loss = loss_act + loss_place else: if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places_unmasked) loss = loss_act + loss_place_entropy else: loss = loss_act if torch_version < 0.4: acc = calculate_accuracy(outputs_unmasked, targets_unmasked) losses.update(loss.data[0], inputs_unmasked.size(0)) else: act_acc = calculate_accuracy_pt_0_4(outputs_unmasked, targets_unmasked) if opt.is_place_adv: if opt.is_place_soft: _, places_hard_target = torch.max(places_unmasked, 1) place_acc = calculate_accuracy_pt_0_4( outputs_places_unmasked, places_hard_target) else: place_acc = calculate_accuracy_pt_0_4( outputs_places_unmasked, places_unmasked) place_losses.update(loss_place.item(), inputs_unmasked.size(0)) else: place_losses.update(0, inputs_unmasked.size(0)) losses.update(loss.item(), inputs_unmasked.size(0)) act_losses.update(loss_act.item(), inputs_unmasked.size(0)) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses.update(loss_place_entropy.item(), inputs_unmasked.size(0)) act_accuracies.update(act_acc, inputs_unmasked.size(0)) if opt.is_place_adv: place_accuracies.update(place_acc, inputs_unmasked.size(0)) else: place_accuracies.update(0, inputs_unmasked.size(0)) # --------------------------------------------------- # Validation on Mask Action Entropy loss (maximize) # --------------------------------------------------- print('num of actual masking_inds = {}/{}'.format( torch.sum(maskings), maskings.shape[0])) if opt.model == 'vgg': inputs_masked = inputs_masked.squeeze() inputs_masked = Variable(inputs_masked, volatile=True) targets_masked = Variable(targets_masked, volatile=True) if opt.is_mask_entropy: if opt.is_place_adv: if opt.is_mask_adv: outputs_masked, outputs_places_masked, outputs_rev_masked = model( inputs_masked) else: outputs_masked, outputs_places_masked = model( inputs_masked) else: if opt.is_mask_adv: outputs_masked, outputs_rev_masked = model(inputs_masked) else: outputs_masked = model(inputs_masked) elif opt.is_mask_cross_entropy: if opt.is_place_adv: outputs_masked, outputs_places_masked, outputs_rev_masked = model( inputs_masked) else: outputs_masked, outputs_rev_masked = model(inputs_masked) if opt.is_mask_entropy: if opt.is_mask_adv: loss_action_entropy = criterions['mask_criterion']( outputs_rev_masked) else: loss_action_entropy = criterions['mask_criterion']( outputs_masked) msk_loss = loss_action_entropy elif opt.is_mask_cross_entropy: loss_action_cross_entropy = criterions['mask_criterion']( outputs_masked, targets_masked) msk_loss = loss_action_cross_entropy if torch_version < 0.4: if opt.is_mask_entropy: if opt.is_mask_adv: acc = calculate_accuracy(outputs_rev_masked, targets_masked) else: acc = calculate_accuracy(outputs_masked, targets_masked) elif opt.is_mask_cross_entropy: acc = calculate_accuracy(outputs_rev_masked, targets_masked) else: if opt.is_mask_adv: msk_act_acc = calculate_accuracy_pt_0_4( outputs_rev_masked, targets_masked) else: msk_act_acc = calculate_accuracy_pt_0_4( outputs_masked, targets_masked) msk_act_losses.update(msk_loss.item(), inputs_masked.size(0)) msk_act_accuracies.update(msk_act_acc, inputs_masked.size(0)) batch_time.update(time.time() - end_time) end_time = time.time() print( 'Val Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Total Loss {loss.val:.4f} ({loss.avg:.4f})\n' 'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t' 'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\t' 'Mask Action Confusion Loss {msk_loss.val:.4f} ({msk_loss.avg:.4f})\n' 'Acc action {act_acc.val:.3f} ({act_acc.avg:.3f})\t' 'Acc place {place_acc.val:.3f} ({place_acc.avg:.3f})\t' 'Acc Mask Action {msk_act_acc.val:.3f} ({msk_act_acc.avg:.3f})'. format(epoch, i + 1, len(data_loader_unmasked), batch_time=batch_time, data_time=data_time, loss=losses, loss_act=act_losses, loss_place=place_losses, msk_loss=msk_act_losses, act_acc=act_accuracies, place_acc=place_accuracies, msk_act_acc=msk_act_accuracies)) logger.log({ 'epoch': epoch, 'loss total': losses.avg, 'loss act': act_losses.avg, 'loss place': place_losses.avg, 'acc act': act_accuracies.avg, 'acc place': place_accuracies.avg }) if tb_writer is not None: tb_writer.add_scalar('val/loss/total', losses.avg, epoch) tb_writer.add_scalar('val/loss/action', act_losses.avg, epoch) tb_writer.add_scalar('val/loss/place', place_losses.avg, epoch) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: tb_writer.add_scalar('val/loss/place_entropy', place_entropy_losses.avg, epoch) tb_writer.add_scalar('val/acc/top1_acc_action', act_accuracies.avg, epoch) tb_writer.add_scalar('val/acc/top1_acc_place', place_accuracies.avg, epoch) tb_writer.add_scalar('val/msk_loss/msk_action', msk_act_losses.avg, epoch) tb_writer.add_scalar('val/msk_acc/top1_msk_acc_action', msk_act_accuracies.avg, epoch) return losses.avg
def main_baseline_model(opt): torch.manual_seed(opt.manual_seed) np.random.seed(opt.manual_seed) model, parameters = generate_model(opt) print(model) criterion = nn.CrossEntropyLoss() if not opt.no_cuda: criterion = criterion.cuda() if opt.no_mean_norm and not opt.std_norm: norm_method = Normalize([0, 0, 0], [1, 1, 1]) elif not opt.std_norm: norm_method = Normalize(opt.mean, [1, 1, 1]) else: norm_method = Normalize(opt.mean, opt.std) if not opt.no_train: assert opt.train_crop in ['random', 'corner', 'center'] if opt.train_crop == 'random': crop_method = MultiScaleRandomCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'corner': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size) elif opt.train_crop == 'center': crop_method = MultiScaleCornerCrop(opt.scales, opt.sample_size, crop_positions=['c']) spatial_transform = Compose([ crop_method, RandomHorizontalFlip(), ToTensor(opt.norm_value), norm_method ]) temporal_transform = TemporalRandomCrop(opt.sample_duration) target_transform = ClassLabel() training_data = get_training_set(opt, spatial_transform, temporal_transform, target_transform) train_loader = torch.utils.data.DataLoader(training_data, batch_size=opt.batch_size, shuffle=True, num_workers=opt.n_threads, pin_memory=True) train_logger = Logger(os.path.join(opt.result_path, 'train.log'), ['epoch', 'loss', 'acc', 'lr']) train_batch_logger = Logger( os.path.join(opt.result_path, 'train_batch.log'), ['epoch', 'batch', 'iter', 'loss', 'acc', 'lr']) if opt.nesterov: dampening = 0 else: dampening = opt.dampening if opt.dataset in [ 'kinetics_adv', 'kinetics_bkgmsk', 'kinetics_adv_msk' ]: first_optimizer = optim.SGD( parameters[1], # new parameters only lr=opt.new_layer_lr, momentum=opt.momentum, dampening=dampening, weight_decay=opt.weight_decay, nesterov=opt.nesterov) first_scheduler = lr_scheduler.ReduceLROnPlateau( first_optimizer, 'min', patience=opt.lr_patience) second_optimizer = optim.SGD( [ { 'params': parameters[0] }, # pretrained parameters { 'params': parameters[1] } # new parameters ], lr=opt.learning_rate, momentum=opt.momentum, dampening=dampening, weight_decay=opt.weight_decay, nesterov=opt.nesterov) second_scheduler = lr_scheduler.ReduceLROnPlateau( second_optimizer, 'min', patience=opt.lr_patience) else: optimizer = optim.SGD(parameters, lr=opt.learning_rate, momentum=opt.momentum, dampening=dampening, weight_decay=opt.weight_decay, nesterov=opt.nesterov) scheduler = lr_scheduler.ReduceLROnPlateau( optimizer, 'min', patience=opt.lr_patience) if not opt.no_val: spatial_transform = Compose([ Scale(opt.sample_size), CenterCrop(opt.sample_size), ToTensor(opt.norm_value), norm_method ]) temporal_transform = LoopPadding(opt.sample_duration) target_transform = ClassLabel() validation_data = get_validation_set(opt, spatial_transform, temporal_transform, target_transform) val_loader = torch.utils.data.DataLoader(validation_data, batch_size=opt.val_batch_size, shuffle=False, num_workers=opt.n_threads, pin_memory=True) val_logger = Logger(os.path.join(opt.result_path, 'val.log'), ['epoch', 'loss', 'acc']) if opt.resume_path: print('loading checkpoint {}'.format(opt.resume_path)) checkpoint = torch.load(opt.resume_path) assert opt.arch == checkpoint['arch'] opt.begin_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) if not opt.no_train: optimizer.load_state_dict(checkpoint['optimizer']) print('run') tr_btime_avg = AverageMeter() tr_dtime_avg = AverageMeter() val_btime_avg = AverageMeter() val_dtime_avg = AverageMeter() for i in range(opt.begin_epoch, opt.n_epochs + 1): if opt.dataset in [ 'kinetics_adv', 'kinetics_bkgmsk', 'kinetics_adv_msk' ]: if i < opt.warm_up_epochs: optimizer = first_optimizer scheduler = first_scheduler else: optimizer = second_optimizer scheduler = second_scheduler if not opt.no_train: tr_btime, tr_dtime = train_epoch(i, train_loader, model, criterion, optimizer, opt, train_logger, train_batch_logger) if not opt.no_val: validation_loss, val_btime, val_dtime = val_epoch( i, val_loader, model, criterion, opt, val_logger) if not opt.no_train and not opt.no_val: scheduler.step(validation_loss) tr_btime_avg.update(tr_btime) tr_dtime_avg.update(tr_dtime) val_btime_avg.update(val_btime) val_dtime_avg.update(val_dtime) print('One epoch tr btime = {:.2f}sec, tr dtime = {:.2f}'.format( tr_btime_avg.avg, tr_dtime_avg.avg)) print('One epoch val btime = {:.2f}sec, val dtime = {:.2f}'.format( val_btime_avg.avg, val_dtime_avg.avg)) sys.stdout.flush() if opt.test: spatial_transform = Compose([ Scale(int(opt.sample_size / opt.scale_in_test)), CornerCrop(opt.sample_size, opt.crop_position_in_test), ToTensor(opt.norm_value), norm_method ]) temporal_transform = LoopPadding(opt.sample_duration) target_transform = VideoID() test_data = get_test_set(opt, spatial_transform, temporal_transform, target_transform) test_loader = torch.utils.data.DataLoader( test_data, batch_size=opt.val_batch_size, shuffle=False, num_workers=opt.n_threads, pin_memory=True) test(test_loader, model, opt, test_data.class_names)
def val_adv_epoch(epoch, data_loader, model, criterions, opt, logger, tb_writer=None): print('validation at epoch {}'.format(epoch)) model.eval() # pytroch version check torch_version = float(torch.__version__[:3]) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() act_losses = AverageMeter() place_losses = AverageMeter() if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses = AverageMeter() act_accuracies = AverageMeter() place_accuracies = AverageMeter() end_time = time.time() for i, (inputs, targets, places) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: targets = targets.cuda() places = places.cuda() if opt.model == 'vgg': inputs = inputs.squeeze() inputs = Variable(inputs, volatile=True) targets = Variable(targets, volatile=True) if opt.is_place_adv: outputs, outputs_places = model(inputs) else: outputs = model(inputs) loss_act = criterions['action_cross_entropy'](outputs, targets) if opt.is_place_adv: loss_place = criterions['places_cross_entropy'](outputs_places, places) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places) loss = loss_act + loss_place + loss_place_entropy else: loss = loss_act + loss_place else: if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places) loss = loss_act + loss_place_entropy else: loss = loss_act if torch_version < 0.4: acc = calculate_accuracy(outputs, targets) losses.update(loss.data[0], inputs.size(0)) else: act_acc = calculate_accuracy_pt_0_4(outputs, targets) if opt.is_place_adv: if opt.is_place_soft: _, places_hard_target = torch.max(places, 1) place_acc = calculate_accuracy_pt_0_4( outputs_places, places_hard_target) else: place_acc = calculate_accuracy_pt_0_4( outputs_places, places) place_losses.update(loss_place.item(), inputs.size(0)) else: place_losses.update(0, inputs.size(0)) losses.update(loss.item(), inputs.size(0)) act_losses.update(loss_act.item(), inputs.size(0)) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses.update(loss_place_entropy.item(), inputs.size(0)) act_accuracies.update(act_acc, inputs.size(0)) if opt.is_place_adv: place_accuracies.update(place_acc, inputs.size(0)) else: place_accuracies.update(0, inputs.size(0)) batch_time.update(time.time() - end_time) end_time = time.time() print('Val Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\n' 'Total Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t' 'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\n' 'Acc action {act_acc.val:.3f} ({act_acc.avg:.3f})\t' 'Acc place {place_acc.val:.3f} ({place_acc.avg:.3f})'.format( epoch, i + 1, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, loss_act=act_losses, loss_place=place_losses, act_acc=act_accuracies, place_acc=place_accuracies)) logger.log({ 'epoch': epoch, 'loss total': losses.avg, 'loss act': act_losses.avg, 'loss place': place_losses.avg, 'acc act': act_accuracies.avg, 'acc place': place_accuracies.avg }) if tb_writer is not None: tb_writer.add_scalar('val/loss/total', losses.avg, epoch) tb_writer.add_scalar('val/loss/action', act_losses.avg, epoch) tb_writer.add_scalar('val/loss/place', place_losses.avg, epoch) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: tb_writer.add_scalar('val/loss/place_entropy', place_entropy_losses.avg, epoch) tb_writer.add_scalar('val/acc/top1_acc_action', act_accuracies.avg, epoch) tb_writer.add_scalar('val/acc/top1_acc_place', place_accuracies.avg, epoch) return losses.avg
def test(val_loader, model, criterion, epoch, use_cuda): global best_acc batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() top5 = AverageMeter() # switch to evaluate mode model.eval() torch.set_grad_enabled(False) end = time.time() bar = Bar('Processing', max=len(val_loader)) for batch_idx, (inputs, targets) in enumerate(val_loader): # measure data loading time data_time.update(time.time() - end) if use_cuda: inputs, targets = inputs.cuda(), targets.cuda() inputs, targets = torch.autograd.Variable(inputs, volatile=True), torch.autograd.Variable(targets) # compute output outputs = model(inputs) loss = criterion(outputs, targets) # measure accuracy and record loss prec1, prec5 = accuracy(outputs.data, targets.data, topk=(1, 5)) # losses.update(loss.data[0], inputs.size(0)) losses.update(loss.data, inputs.size(0)) #top1.update(prec1[0], inputs.size(0)) top1.update(prec1, inputs.size(0)) #top5.update(prec5[0], inputs.size(0)) top5.update(prec5, inputs.size(0)) # measure elapsed time batch_time.update(time.time() - end) end = time.time() # plot progress bar.suffix = '({batch}/{size}) Data: {data:.3f}s | Batch: {bt:.3f}s | Total: {total:} | ETA: {eta:} | Loss: {loss:.4f} | top1: {top1: .4f} | top5: {top5: .4f}'.format( batch=batch_idx + 1, size=len(val_loader), data=data_time.avg, bt=batch_time.avg, total=bar.elapsed_td, eta=bar.eta_td, loss=losses.avg, top1=top1.avg, top5=top5.avg, ) bar.next() print(bar.suffix) bar.finish() return (losses.avg, top1.avg)
def train_adv_epoch(epoch, data_loader, model, criterions, optimizer, opt, epoch_logger, batch_logger, warm_up_epochs, tb_writer=None): print('train at epoch {}'.format(epoch)) model.train() # pytroch version check torch_version = float(torch.__version__[:3]) batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() act_losses = AverageMeter() place_losses = AverageMeter() if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses = AverageMeter() act_accuracies = AverageMeter() place_accuracies = AverageMeter() end_time = time.time() for i, (inputs, targets, places) in enumerate(data_loader): data_time.update(time.time() - end_time) if not opt.no_cuda: targets = targets.cuda(async=True) places = places.cuda(async=True) if opt.model == 'vgg': inputs = inputs.squeeze() inputs = Variable(inputs) targets = Variable(targets) if opt.is_place_adv: outputs, outputs_places = model(inputs) else: outputs = model(inputs) loss_act = criterions['action_cross_entropy'](outputs, targets) if opt.is_place_adv: loss_place = criterions['places_cross_entropy'](outputs_places, places) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places) loss = loss_act + loss_place + opt.weight_entropy_loss * loss_place_entropy else: loss = loss_act + loss_place else: if opt.is_place_entropy and epoch >= opt.warm_up_epochs: loss_place_entropy = criterions['places_entropy']( outputs_places) loss = loss_act + opt.weight_entropy_loss * loss_place_entropy else: loss = loss_act if torch_version < 0.4: acc = calculate_accuracy(outputs, targets) losses.update(loss.data[0], inputs.size(0)) else: act_acc = calculate_accuracy_pt_0_4(outputs, targets) if opt.is_place_adv: if opt.is_place_soft: _, places_hard_target = torch.max(places, 1) place_acc = calculate_accuracy_pt_0_4( outputs_places, places_hard_target) else: place_acc = calculate_accuracy_pt_0_4( outputs_places, places) place_losses.update(loss_place.item(), inputs.size(0)) else: place_losses.update(0, inputs.size(0)) losses.update(loss.item(), inputs.size(0)) act_losses.update(loss_act.item(), inputs.size(0)) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: place_entropy_losses.update(loss_place_entropy.item(), inputs.size(0)) act_accuracies.update(act_acc, inputs.size(0)) if opt.is_place_adv: place_accuracies.update(place_acc, inputs.size(0)) else: place_accuracies.update(0, inputs.size(0)) optimizer.zero_grad() loss.backward() optimizer.step() batch_time.update(time.time() - end_time) end_time = time.time() batch_logger.log({ 'epoch': epoch, 'batch': i + 1, 'iter': (epoch - 1) * len(data_loader) + (i + 1), 'loss total': losses.val, 'loss act': act_losses.val, 'loss place': place_losses.val, 'acc act': act_accuracies.val, 'acc place': place_accuracies.val, 'lr': optimizer.param_groups[0]['lr'] }) print('Train Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\n' 'Total Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Action Loss {loss_act.val:.4f} ({loss_act.avg:.4f})\t' 'Place Loss {loss_place.val:.4f} ({loss_place.avg:.4f})\n' 'Acc action {act_acc.val:.3f} ({act_acc.avg:.3f})\t' 'Acc place {place_acc.val:.3f} ({place_acc.avg:.3f})'.format( epoch, i + 1, len(data_loader), batch_time=batch_time, data_time=data_time, loss=losses, loss_act=act_losses, loss_place=place_losses, act_acc=act_accuracies, place_acc=place_accuracies)) epoch_logger.log({ 'epoch': epoch, 'loss total': losses.avg, 'loss act': act_losses.avg, 'loss place': place_losses.avg, 'acc act': act_accuracies.avg, 'acc place': place_accuracies.avg, 'lr': optimizer.param_groups[0]['lr'] }) if epoch % opt.checkpoint == 0: save_file_path = os.path.join(opt.result_path, 'save_{}.pth'.format(epoch)) states = { 'epoch': epoch + 1, 'arch': opt.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), } torch.save(states, save_file_path) if tb_writer is not None: tb_writer.add_scalar('train/loss/total', losses.avg, epoch) tb_writer.add_scalar('train/loss/action', act_losses.avg, epoch) tb_writer.add_scalar('train/loss/place', place_losses.avg, epoch) if opt.is_place_entropy and epoch >= opt.warm_up_epochs: tb_writer.add_scalar('train/loss/place_entropy', place_entropy_losses.avg, epoch) tb_writer.add_scalar('train/acc/top1_acc_action', act_accuracies.avg, epoch) tb_writer.add_scalar('train/acc/top1_acc_place', place_accuracies.avg, epoch) if epoch < warm_up_epochs: tb_writer.add_scalar('train/lr_new', optimizer.param_groups[0]['lr'], epoch) else: tb_writer.add_scalar('train/lr_pretrained', optimizer.param_groups[0]['lr'], epoch) tb_writer.add_scalar('train/lr_new', optimizer.param_groups[-1]['lr'], epoch)