def validate(self, loader, model, criterion, epoch, args): timer = Timer() losses = AverageMeter() top1 = AverageMeter() wtop1 = AverageMeter() alloutputs = [] metrics = {} # switch to evaluate mode model.eval() def part(x): return itertools.islice(x, int(len(x) * args.val_size)) for i, x in enumerate(part(loader)): inputs, target, meta = parse(x) output, loss, weights = forward(inputs, target, model, criterion, meta['id'], train=False) prec1 = triplet_accuracy(output, target) wprec1 = triplet_accuracy(output, target, weights) losses.update(loss.item(), inputs[0].size(0)) top1.update(prec1, inputs[0].size(0)) wtop1.update(wprec1, inputs[0].size(0)) alloutputs.extend( zip([(x.item(), y.item()) for x, y in zip(*output)], target, weights)) timer.tic() if i % args.print_freq == 0: print('[{name}] Test [{epoch}]: [{0}/{1} ({2})]\t' 'Time {timer.val:.3f} ({timer.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'WAcc@1 {wtop1.val:.3f} ({wtop1.avg:.3f})\t'.format( i, int(len(loader) * args.val_size), len(loader), name=args.name, timer=timer, loss=losses, top1=top1, epoch=epoch, wtop1=wtop1)) metrics.update(triplet_allk(*zip(*alloutputs))) metrics.update({'top1val': top1.avg, 'wtop1val': wtop1.avg}) print( ' * Acc@1 {top1val:.3f} \t WAcc@1 {wtop1val:.3f}' '\n topk1: {topk1:.3f} \t topk2: {topk2:.3f} \t ' 'topk5: {topk5:.3f} \t topk10: {topk10:.3f} \t topk50: {topk50:.3f}' .format(**metrics)) return metrics
def train(self, loader, model, criterion, optimizer, epoch, args): adjust_learning_rate(args.lr, args.lr_decay_rate, optimizer, epoch) timer = Timer() data_time = AverageMeter() losses = AverageMeter() top1 = AverageMeter() wtop1 = AverageMeter() metrics = {} # switch to train mode model.train() optimizer.zero_grad() def part(x): return itertools.islice(x, int(len(x) * args.train_size)) for i, x in enumerate(part(loader)): inputs, target, meta = parse(x) data_time.update(timer.thetime() - timer.end) output, loss, weights = forward(inputs, target, model, criterion, meta['id']) prec1 = triplet_accuracy(output, target) wprec1 = triplet_accuracy(output, target, weights) losses.update(loss.item(), inputs[0].size(0)) top1.update(prec1, inputs[0].size(0)) wtop1.update(wprec1, inputs[0].size(0)) loss.backward() if i % args.accum_grad == args.accum_grad - 1: print('updating parameters') optimizer.step() optimizer.zero_grad() timer.tic() if i % args.print_freq == 0: print('[{name}] Epoch: [{0}][{1}/{2}({3})]\t' 'Time {timer.val:.3f} ({timer.avg:.3f})\t' 'Data {data_time.val:.3f} ({data_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Acc@1 {top1.val:.3f} ({top1.avg:.3f})\t' 'WAcc@1 {wtop1.val:.3f} ({wtop1.avg:.3f})\t'.format( epoch, i, int(len(loader) * args.train_size), len(loader), name=args.name, timer=timer, data_time=data_time, loss=losses, top1=top1, wtop1=wtop1)) metrics.update({'top1': top1.avg, 'wtop1': wtop1.avg}) return metrics
def fine_tune_train_and_val(args, recorder): # = global lowest_val_loss, best_prec1 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # close the warning torch.manual_seed(1) cudnn.benchmark = True timer = Timer() # == dataset config== num_class, data_length, image_tmpl = ft_data_config(args) train_transforms, test_transforms, eval_transforms = ft_augmentation_config( args) train_data_loader, val_data_loader, _, _, _, _ = ft_data_loader_init( args, data_length, image_tmpl, train_transforms, test_transforms, eval_transforms) # == model config== model = ft_model_config(args, num_class) recorder.record_message('a', '=' * 100) recorder.record_message('a', '-' * 40 + 'finetune' + '-' * 40) recorder.record_message('a', '=' * 100) # == optim config== train_criterion, val_criterion, optimizer = ft_optim_init(args, model) # == data augmentation(self-supervised) config== tc = TC(args) # == train and eval== print('*' * 70 + 'Step2: fine tune' + '*' * 50) for epoch in range(args.ft_start_epoch, args.ft_epochs): timer.tic() ft_adjust_learning_rate(optimizer, args.ft_lr, epoch, args.ft_lr_steps) train_prec1, train_loss = train(args, tc, train_data_loader, model, train_criterion, optimizer, epoch, recorder) # train_prec1, train_loss = random.random() * 100, random.random() recorder.record_ft_train(train_loss / 5.0, train_prec1 / 100.0) if (epoch + 1) % args.ft_eval_freq == 0: val_prec1, val_loss = validate(args, tc, val_data_loader, model, val_criterion, recorder) # val_prec1, val_loss = random.random() * 100, random.random() recorder.record_ft_val(val_loss / 5.0, val_prec1 / 100.0) is_best = val_prec1 > best_prec1 best_prec1 = max(val_prec1, best_prec1) checkpoint = { 'epoch': epoch + 1, 'arch': "i3d", 'state_dict': model.state_dict(), 'best_prec1': best_prec1 } recorder.save_ft_model(checkpoint, is_best) timer.toc() left_time = timer.average_time * (args.ft_epochs - epoch) message = "Step2: fine tune best_prec1 is: {} left time is : {} now is : {}".format( best_prec1, timer.format(left_time), datetime.now()) print(message) recorder.record_message('a', message) return recorder.filename
def validate_egovideo(self, loader, model, epoch, args): """ Run video-level validation on the Charades ego test set""" timer = Timer() outputs, gts, ids = [], [], [] outputsw = [] metrics = {} # switch to evaluate mode model.eval() for i, x in enumerate(loader): inp, target, meta = parse(x) target = target.long().cuda(async=True) assert target[0, :].eq(target[1, :]).all(), "val_video not synced" input_var = torch.autograd.Variable(inp.cuda(), volatile=True) output, w_x, w_z = model(input_var) output = torch.nn.Softmax(dim=1)(output) sw_x = torch.nn.Softmax(dim=0)(w_x) * w_x.shape[0] sw_x = (sw_x - sw_x.mean()) / sw_x.std() scale = torch.clamp(1 + (sw_x - 1) * 0.05, 0, 100) print('scale min: {}\t max: {}\t std: {}'.format( scale.min().data[0], scale.max().data[0], scale.std().data[0])) scale = torch.clamp(scale, 0, 100) scale *= scale.shape[0] / scale.sum() outputw = output * scale.unsqueeze(1) # store predictions output_video = output.mean(dim=0) outputs.append(output_video.data.cpu().numpy()) outputsw.append(outputw.mean(dim=0).data.cpu().numpy()) gts.append(target[0, :]) ids.append(meta['id'][0]) timer.tic() if i % args.print_freq == 0: print('Test2: [{0}/{1}]\t' 'Time {timer.val:.3f} ({timer.avg:.3f})'.format( i, len(loader), timer=timer)) # mAP, _, ap = meanap.map(np.vstack(outputs), np.vstack(gts)) mAP, _, ap = meanap.charades_nanmap(np.vstack(outputs), np.vstack(gts)) mAPw, _, _ = meanap.charades_nanmap(np.vstack(outputsw), np.vstack(gts)) metrics['mAPego'] = mAP metrics['mAPegow'] = mAPw print(ap) print(' * mAPego {mAPego:.3f} \t mAPegow {mAPegow:.3f}'.format( **metrics)) submission_file(ids, outputs, '{}/egoepoch_{:03d}.txt'.format(args.cache, epoch + 1)) return metrics
def validate_video(self, loader, model, epoch, args): """ Run video-level validation on the Charades test set""" timer = Timer() outputs, gts, ids = [], [], [] metrics = {} # switch to evaluate mode model.eval() for i, x in enumerate(loader): inputs, target, meta = parse(x) target = target.long().cuda(async=True) assert target[0, :].eq(target[1, :]).all(), "val_video not synced" input_vars = [ torch.autograd.Variable(inp.cuda(), volatile=True) for inp in inputs ] output = model( *input_vars)[-1] # classification should be last output output = torch.nn.Softmax(dim=1)(output) # store predictions output_video = output.mean(dim=0) outputs.append(output_video.data.cpu().numpy()) gts.append(target[0, :]) ids.append(meta['id'][0]) timer.tic() if i % args.print_freq == 0: print('Test2: [{0}/{1}]\t' 'Time {timer.val:.3f} ({timer.avg:.3f})'.format( i, len(loader), timer=timer)) # mAP, _, ap = meanap.map(np.vstack(outputs), np.vstack(gts)) mAP, _, ap = meanap.charades_map(np.vstack(outputs), np.vstack(gts)) metrics['mAP'] = mAP print(ap) print(' * mAP {:.3f}'.format(mAP)) submission_file(ids, outputs, '{}/epoch_{:03d}.txt'.format(args.cache, epoch + 1)) return metrics
def pretext_train(args, recorder): if args.gpus is not None: print("Use GPU: {} for pretext training".format(args.gpus)) num_class, data_length, image_tmpl = pt_data_config(args) # print("tp_length is: ", data_length) train_transforms, test_transforms, eval_transforms = pt_augmentation_config( args) train_loader, val_loader, eval_loader, train_samples, val_samples, eval_samples = pt_data_loader_init( args, data_length, image_tmpl, train_transforms, test_transforms, eval_transforms) n_data = len(train_loader) model, model_ema = pt_model_config(args, num_class) # == optim config== contrast, criterion, optimizer = pt_optim_init(args, model, n_data) model = model.cuda() # == load weights == model, model_ema = pt_load_weight(args, model, model_ema, optimizer, contrast) if args.pt_method in ['dsm', 'moco']: model_ema = model_ema.cuda() # copy weights from `model' to `model_ema' moment_update(model, model_ema, 0) cudnn.benchmark = True # optionally resume from a checkpoint args.start_epoch = 1 # ==================================== our data augmentation method================================= if args.pt_method in ['dsm', 'dsm_triplet']: pos_aug = GenPositive() neg_aug = GenNegative() # =======================================add message ===================== recorder.record_message('a', '=' * 100) recorder.record_message('a', '-' * 40 + 'pretrain' + '-' * 40) recorder.record_message('a', '=' * 100) # ====================update lr_decay from str to numpy========= iterations = args.pt_lr_decay_epochs.split(',') args.pt_lr_decay_epochs = list([]) for it in iterations: args.pt_lr_decay_epochs.append(int(it)) timer = Timer() # routine print('*' * 70 + 'Step1: pretrain' + '*' * 20 + '*' * 50) for epoch in range(args.pt_start_epoch, args.pt_epochs + 1): timer.tic() pt_adjust_learning_rate(epoch, args, optimizer) print("==> training...") time1 = time.time() if args.pt_method == "moco": loss, prob = train_moco(epoch, train_loader, model, model_ema, contrast, criterion, optimizer, args, recorder) elif args.pt_method == "dsm": loss, prob = train_dsm(epoch, train_loader, model, model_ema, contrast, criterion, optimizer, args, pos_aug, neg_aug, recorder) # loss, prob = epoch * 0.01, 0.02*epoch elif args.pt_method == "dsm_triplet": loss = train_dsm_triplet(epoch, train_loader, model, optimizer, args, pos_aug, neg_aug, recorder) else: Exception("Not support method now!") recorder.record_pt_train(loss) time2 = time.time() print('epoch {}, total time {:.2f}'.format(epoch, time2 - time1)) timer.toc() left_time = timer.average_time * (args.pt_epochs - epoch) message = "Step1: pretrain now loss is: {} left time is : {} now is: {}".format( loss, timer.format(left_time), datetime.now()) print(message) recorder.record_message('a', message) state = { 'opt': args, 'model': model.state_dict(), 'contrast': contrast.state_dict(), 'optimizer': optimizer.state_dict(), 'epoch': epoch, } recorder.save_pt_model(args, state, epoch) print("finished pretrain, the trained model is record in: {}".format( recorder.pt_checkpoint)) return recorder.pt_checkpoint
def inference_mean_exemplar( model, current_epoch, current_iter, local_rank, data_loader, dataset_name, device="cuda", max_instance=3200, mute=False, ): model.train(False) # convert to a torch.device for efficiency device = torch.device(device) if not mute: logger = logging.getLogger("maskrcnn_benchmark.inference") logger.info("Start evaluation") total_timer = Timer() inference_timer = Timer() total_timer.tic() torch.cuda.empty_cache() if not mute: pbar = tqdm(total=len(data_loader), desc="Validation in progress") with torch.no_grad(): all_pred_obj, all_truth_obj, all_pred_attr, all_truth_attr = [], [], [], [] obj_loss_all, attr_loss_all = 0, 0 cnt = 0 for iteration, out_dict in enumerate(data_loader): if type(max_instance) is int: if iteration == max_instance // model.cfg.EXTERNAL.BATCH_SIZE: break if type(max_instance) is float: if iteration > max_instance * len( data_loader) // model.cfg.EXTERNAL.BATCH_SIZE: break # print(iteration) images = torch.stack(out_dict['images']) obj_labels = torch.cat(out_dict['object_labels'], -1) attr_labels = torch.cat(out_dict['attribute_labels'], -1) cropped_image = torch.stack(out_dict['cropped_image']) images = images.to(device) obj_labels = obj_labels.to(device) attr_labels = attr_labels.to(device) cropped_image = cropped_image.to(device) # loss_dict = model(images, targets) pred_obj = model.mean_of_exemplar_classify(cropped_image) all_pred_obj.extend(to_list(pred_obj)) all_truth_obj.extend(to_list(obj_labels)) cnt += 1 if not mute: pbar.update(1) obj_f1 = f1_score(all_truth_obj, all_pred_obj, average='micro') #attr_f1 = f1_score(all_truth_attr, all_pred_attr, average='micro') obj_loss_all /= (cnt + 1e-10) # wait for all processes to complete before measuring the time total_time = total_timer.toc() model.train(True) return obj_f1, 0, len(all_truth_obj)
def inference(model, current_epoch, current_iter, local_rank, data_loader, dataset_name, device="cuda", max_instance=3200, mute=False, verbose_return=False): model.train(False) # convert to a torch.device for efficiency device = torch.device(device) if not mute: logger = logging.getLogger("maskrcnn_benchmark.inference") logger.info("Start evaluation") total_timer = Timer() total_timer.tic() torch.cuda.empty_cache() if not mute: pbar = tqdm(total=len(data_loader), desc="Validation in progress") def to_list(tensor): return tensor.cpu().numpy().tolist() with torch.no_grad(): all_pred_obj, all_truth_obj, all_pred_attr, all_truth_attr = [], [], [], [] all_image_ids, all_boxes = [], [] all_pred_attr_prob = [] all_raws = [] obj_loss_all, attr_loss_all = 0, 0 cnt = 0 for iteration, out_dict in enumerate(data_loader): if type(max_instance) is int: if iteration == max_instance // model.cfg.EXTERNAL.BATCH_SIZE: break if type(max_instance) is float: if iteration > max_instance * len( data_loader) // model.cfg.EXTERNAL.BATCH_SIZE: break # print(iteration) if verbose_return: all_image_ids.extend(out_dict['image_ids']) all_boxes.extend(out_dict['gt_bboxes']) all_raws.extend(out_dict['raw']) ret_dict = inference_step(model, out_dict, device) loss_attr, loss_obj, attr_score, obj_score = ret_dict.get('attr_loss', None), \ ret_dict.get('obj_loss', None), \ ret_dict.get('attr_score', None), \ ret_dict.get('obj_score', None) if loss_attr is not None: attr_loss_all += loss_attr.item() pred_attr_prob, pred_attr = ret_dict[ 'pred_attr_prob'], ret_dict['pred_attr'] all_pred_attr.extend(to_list(pred_attr)) all_truth_attr.extend(to_list(ret_dict['attr_labels'])) all_pred_attr_prob.extend(to_list(pred_attr_prob)) if loss_obj is not None: obj_loss_all += loss_obj.item() _, pred_obj = obj_score.max(-1) all_pred_obj.extend(to_list(pred_obj)) all_truth_obj.extend(to_list(ret_dict['obj_labels'])) cnt += 1 if not mute: pbar.update(1) obj_f1 = f1_score(all_truth_obj, all_pred_obj, average='micro') attr_f1 = f1_score(all_truth_attr, all_pred_attr, average='micro') obj_loss_all /= (cnt + 1e-10) attr_loss_all /= (cnt + 1e-10) if not mute: logger.info( 'Epoch: {}\tIteration: {}\tObject f1: {}\tAttr f1:{}\tObject loss:{}\tAttr loss:{}' .format(current_epoch, current_iter, obj_f1, attr_f1, obj_loss_all, attr_loss_all)) #compute_on_dataset(model, data_loader, local_rank, device, inference_timer, output_file) # wait for all processes to complete before measuring the time total_time = total_timer.toc() model.train(True) if not verbose_return: return obj_f1, attr_f1, len(all_truth_attr) else: return obj_f1, attr_f1, all_pred_attr, all_truth_attr, all_pred_obj, all_truth_obj, all_image_ids, all_boxes, \ all_pred_attr_prob, all_raws
def train_and_eval(args): # = global lowest_val_loss, best_prec1 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # close the warning os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus torch.manual_seed(1) cudnn.benchmark = True timer = Timer() recorder = Record(args) # == dataset config== num_class, data_length, image_tmpl = data_config(args) train_transforms, test_transforms = augmentation_config(args) train_data_loader, val_data_loader = data_loader_init( args, data_length, image_tmpl, train_transforms, test_transforms) # == model config== models = [] optimizers = [] for i in range(args.mutual_num): model = model_config(args, num_class) models.append(model) recorder.record_message('a', '=' * 100) recorder.record_message('a', str(model.module)) recorder.record_message('a', '=' * 100) # == optim config== for i in range(args.mutual_num): train_criterion, val_criterion, optimizer = optim_init(args, model) optimizers.append(optimizer) # == data augmentation(self-supervised) config== tc = TC(args) # == train and eval== for epoch in range(args.start_epoch, args.epochs): timer.tic() for i in range(args.mutual_num): adjust_learning_rate(optimizers[i], args.lr, epoch, args.lr_steps) if args.eval_indict == 'acc': train_prec1, train_loss = train(args, tc, train_data_loader, models, train_criterion, optimizers, epoch, recorder) # train_prec1, train_loss = random.random() * 100, random.random() recorder.record_train(train_loss / 5.0, train_prec1 / 100.0) else: train_loss = train(args, tc, train_data_loader, models, train_criterion, optimizers, epoch, recorder) # train_prec1, train_loss = random.random() * 100, random.random() recorder.record_train(train_loss) if (epoch + 1) % args.eval_freq == 0: if args.eval_indict == 'acc': val_prec1, val_loss = validate(args, tc, val_data_loader, models, val_criterion, recorder) # val_prec1, val_loss = random.random() * 100, random.random() recorder.record_val(val_loss / 5.0, val_prec1 / 100.0) is_best = val_prec1 > best_prec1 best_prec1 = max(val_prec1, best_prec1) checkpoint = { 'epoch': epoch + 1, 'arch': "i3d", 'state_dict': model.state_dict(), 'best_prec1': best_prec1 } else: val_loss = validate(args, tc, val_data_loader, models, val_criterion, recorder) # val_loss = random.random() # val_prec1, val_loss = random.random() * 100, random.random() recorder.record_val(val_loss) is_best = val_loss < lowest_val_loss lowest_val_loss = min(val_loss, lowest_val_loss) checkpoint = { 'epoch': epoch + 1, 'arch': "i3d", 'state_dict': model.state_dict(), 'lowest_val': lowest_val_loss } recorder.save_model(checkpoint, is_best) timer.toc() left_time = timer.average_time * (args.epochs - epoch) if args.eval_indict == 'acc': message = "best_prec1 is: {} left time is : {}".format( best_prec1, timer.format(left_time)) else: message = "lowest_val_loss is: {} left time is : {}".format( lowest_val_loss, timer.format(left_time)) print(message) recorder.record_message('a', message) # return recorder.best_name return recorder.filename