def eval_gcr(model, criterion, valloader, device, epoch, log_interval, writer, args): batch_time = AverageMeter() data_time = AverageMeter() avg_loss1 = AverageMeter() avg_loss2 = AverageMeter() avg_acc1 = AverageMeter() avg_acc2 = AverageMeter() statistic = [] # Create recorder averagers = [avg_loss1, avg_loss2, avg_acc1, avg_acc2] names = ['val loss1','val loss2','val acc1','val acc2'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() for i, batch in enumerate(valloader): with torch.no_grad(): # measure data loading time recoder.data_tok() # get the inputs and labels data, lab = [_.to(device) for _ in batch] # forward p = args.shot * args.test_way data_shot = data[:p] data_query = data[p:] logits, label, logits2, gt = \ model(data_shot,data_query,lab,mode='eval') # compute the loss loss, loss1, loss2 = criterion(logits, label, logits2, gt) # print('logits: {}'.format(logits)) # print('out: {}'.format(logits.argmax(-1))) # print('label: {}'.format(label)) # compute the metrics acc1 = accuracy(logits, label)[0] acc2 = accuracy(logits2, gt)[0] # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value & account statistic vals = [loss1.item(),loss2.item(),acc1,acc2] recoder.update(vals) statistic.append(acc1.data.cpu().numpy()) if i % log_interval == log_interval-1: recoder.log(epoch,i,len(valloader),mode='Eval') return recoder.get_avg('val acc1'), numpy.array(statistic)
def eval_mn_pn(model, criterion, valloader, device, epoch, log_interval, writer, args): batch_time = AverageMeter() data_time = AverageMeter() avg_loss = AverageMeter() avg_acc = AverageMeter() statistic = [] # Create recorder averagers = [avg_loss,avg_acc] names = ['val loss','val acc'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() for i, batch in enumerate(valloader): with torch.no_grad(): # measure data loading time recoder.data_tok() # get the inputs and labels data, lab = [_.to(device) for _ in batch] # forward p = args.shot * args.test_way data_shot = data[:p] data_query = data[p:] y_pred, label = model(data_shot,data_query,mode='eval') # print('lab: {}'.format(lab.view((args.shot+args.query_val),args.test_way)[0])) # compute the loss loss = criterion(y_pred, label) # print('y_pred: {}'.format(y_pred.argmax(-1))) # print('label: {}'.format(label)) # compute the metrics acc = accuracy(y_pred, label)[0] # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value & account statistic vals = [loss.item(),acc] recoder.update(vals) statistic.append(acc.data.cpu().numpy()) if i % log_interval == log_interval-1: recoder.log(epoch,i,len(valloader),mode='Eval') return recoder.get_avg('val acc'), numpy.array(statistic)
def test_gcr(model, criterion, valloader, device, epoch, log_interval, writer, args, relation): batch_time = AverageMeter() data_time = AverageMeter() avg_loss = AverageMeter() avg_acc = AverageMeter() # Create recorder averagers = [avg_loss, avg_acc] names = ['val loss', 'val acc'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() for i, batch in enumerate(valloader): with torch.no_grad(): # measure data loading time recoder.data_tok() # get the inputs and labels data, lab = [_.to(device) for _ in batch] # forward proto = model.baseModel(data) global_set = torch.cat([model.global_base,model.global_novel]) logits = relation(proto,global_set) # print('logits: ',logits.argmax(-1)) # print('lab: ',lab) # compute the loss loss = criterion(logits, lab) # compute the metrics acc = accuracy(logits, lab)[0] # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value vals = [loss.item(),acc] recoder.update(vals) if i % log_interval == log_interval-1: recoder.log(epoch,i,len(valloader),mode='Test') return recoder.get_avg('val acc')
def eval_cnn(model, criterion, valloader, device, epoch, log_interval, writer, args): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() avg_acc = AverageMeter() averagers = [losses, avg_acc] names = ['val loss','val acc'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() for i, batch in enumerate(valloader): with torch.no_grad(): # measure data loading time recoder.data_tok() # get the data and labels data,lab = [_.to(device) for _ in batch] p = args.shot * args.test_way data_shot = data[:p] data_query = data[p:] input = torch.cat([data_shot,data_query],0) # forward outputs = model(input) # compute the loss loss = criterion(outputs,lab) # compute the metrics acc = accuracy(outputs, lab)[0] # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value vals = [loss.item(),acc] recoder.update(vals) # logging if i==0 or i % log_interval == log_interval-1 or i==len(valloader)-1: recoder.log(epoch,i,len(valloader),mode='Eval') return recoder.get_avg('val acc')
def eval_c3d(model, criterion, valloader, device, epoch, log_interval, writer, eval_samples): batch_time = AverageMeter() data_time = AverageMeter() losses = AverageMeter() avg_top1 = AverageMeter() avg_top5 = AverageMeter() averagers = [losses, avg_top1, avg_top5] names = ['val loss','val top1','val top5'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() for i, batch in enumerate(valloader): with torch.no_grad(): # Reduce the evaluation time !!! if i>eval_samples: break # measure data loading time recoder.data_tok() # get the data and labels data,lab = [_.to(device) for _ in batch] # forward outputs = model(data) # compute the loss loss = criterion(outputs,lab) # compute the metrics top1, top5 = accuracy(outputs, lab, topk=(1,5)) # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value vals = [loss.item(),top1,top5] recoder.update(vals) # logging if i==0 or i % log_interval == log_interval-1 or i==len(valloader)-1: recoder.log(epoch,i,len(valloader),mode='Eval') return recoder.get_avg('val top1')
def test_mn(model, global_proto, criterion, valloader, device, epoch, log_interval, writer, args): batch_time = AverageMeter() data_time = AverageMeter() avg_loss = AverageMeter() avg_acc = AverageMeter() statistic = [] # Create recorder averagers = [avg_loss,avg_acc] names = ['val loss','val acc'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() for i, batch in enumerate(valloader): with torch.no_grad(): # measure data loading time recoder.data_tok() # get the inputs and labels data, lab = [_.to(device) for _ in batch] # forward y_pred = model.gfsl_test(global_proto,data) # compute the loss loss = criterion(y_pred, lab) # compute the metrics acc = accuracy(y_pred, lab)[0] # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value & account statistic vals = [loss.item(),acc] recoder.update(vals) if i % log_interval == log_interval-1: recoder.log(epoch,i,len(valloader),mode='Test') return recoder.get_avg('val acc')
def test_seq2seq(model, criterion, dataloader, device, epoch, log_interval, writer): batch_time = AverageMeter() data_time = AverageMeter() avg_loss = AverageMeter() avg_acc = AverageMeter() avg_wer = AverageMeter() # Create recorder averagers = [avg_loss, avg_acc, avg_wer] names = ['test loss', 'test acc', 'test wer'] recoder = Recorder(averagers, names, writer, batch_time, data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() with torch.no_grad(): for batch_idx, batch in enumerate(dataloader): # measure data loading time recoder.data_tok() # get the data and labels imgs = batch['videos'].cuda() target = batch['annotations'].permute(1, 0).contiguous().cuda() # forward(no teacher forcing) outputs = model(imgs, target, 0) # target: (batch_size, trg len) # outputs: (trg_len, batch_size, output_dim) # skip sos output_dim = outputs.shape[-1] outputs = outputs[1:].view(-1, output_dim) target = target.permute(1, 0)[1:].reshape(-1) # compute the loss loss = criterion(outputs, target) # compute the accuracy prediction = torch.max(outputs, 1)[1] score = accuracy_score(target.cpu().data.squeeze().numpy(), prediction.cpu().data.squeeze().numpy()) # compute wer # prediction: ((trg_len-1)*batch_size) # target: ((trg_len-1)*batch_size) batch_size = imgs.shape[0] prediction = prediction.view(-1, batch_size).permute(1, 0).tolist() target = target.view(-1, batch_size).permute(1, 0).tolist() wers = [] for i in range(batch_size): # add mask(remove padding, eos, sos) prediction[i] = [ item for item in prediction[i] if item not in [0, 1, 2] ] target[i] = [ item for item in target[i] if item not in [0, 1, 2] ] wers.append(wer(target[i], prediction[i])) batch_wer = sum(wers) / len(wers) # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value vals = [loss.item(), score, batch_wer] b = imgs.size(0) recoder.update(vals, count=b) # logging if batch_idx == 0 or batch_idx % log_interval == log_interval - 1 or batch_idx == len( dataloader) - 1: recoder.log(epoch, batch_idx, len(dataloader), mode='Test') return recoder.get_avg('val acc')
def evaluate_confusion_matrix(model, criterion, valloader, device, epoch, log_interval, writer, args, relation, name, category_space='novel', base_class=400,novel_class=100): batch_time = AverageMeter() data_time = AverageMeter() avg_loss = AverageMeter() avg_acc = AverageMeter() # Create recorder averagers = [avg_loss, avg_acc] names = ['val loss', 'val acc'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() num_class = base_class + novel_class cmat = numpy.zeros([num_class,num_class]) for i, batch in enumerate(valloader): with torch.no_grad(): # measure data loading time recoder.data_tok() # get the inputs and labels data, lab = [_.to(device) for _ in batch] # forward proto = model.baseModel(data) global_set = torch.cat([model.global_base,model.global_novel]) logits = relation(proto,global_set) # print('logits: ',logits.argmax(-1)) # print('lab: ',lab) # compute the loss loss = criterion(logits, lab) # compute the metrics acc = accuracy(logits, lab)[0] # compute the confusion matrix predict = logits.argmax(-1) for p,g in zip(predict,lab): cmat[g,p] += 1 # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value vals = [loss.item(),acc] recoder.update(vals) if i % log_interval == log_interval-1: recoder.log(epoch,i,len(valloader),mode='Test') # normalize & print confusion matrix cmat = cmat / cmat.sum(1) if category_space == 'novel': cmat = cmat[base_class:,base_class:] elif category_space == 'base': cmat = cmat[:base_class,:base_class] df = pd.DataFrame(cmat) df.to_csv(name) return recoder.get_avg('val acc')
def eval_maml(model, criterion, valloader, device, epoch, log_interval, writer, args): batch_time = AverageMeter() data_time = AverageMeter() avg_loss = AverageMeter() avg_acc = AverageMeter() # Create recorder averagers = [avg_loss, avg_acc] names = ['val loss','val acc'] recoder = Recorder(averagers,names,writer,batch_time,data_time) # Set evaluation mode model.eval() recoder.tik() recoder.data_tik() # Settings create_graph = (True if args.order == 2 else False) for i, batch in enumerate(valloader): # measure data loading time recoder.data_tok() # get the inputs and labels data, lab = [_.to(device) for _ in batch] # forward # data = data.view( ((args.shot+args.query),args.train_way) + data.size()[-3:] ) # data = data.permute(1,0,2,3,4).contiguous() # data = data.view( (-1,) + data.size()[-3:] ) p = args.shot * args.test_way data_shot = data[:p] data_query = data[p:] data_shape = data.size()[-3:] # Create a fast model using the current meta model weights fast_weights = OrderedDict(model.named_parameters()) # Train the model for `inner_train_steps` iterations for inner_batch in range(args.inner_train_steps): # Perform update of model weights y = create_nshot_task_label(args.test_way, args.shot).to(device) logits = model.functional_forward(data_shot, fast_weights) loss = criterion(logits, y) gradients = torch.autograd.grad(loss, fast_weights.values(), create_graph=create_graph) # Update weights manually fast_weights = OrderedDict( (name, param - args.inner_lr * grad) for ((name, param), grad) in zip(fast_weights.items(), gradients) ) # Do a pass of the model on the validation data from the current task y = create_nshot_task_label(args.test_way, args.query_val).to(device) logits = model.functional_forward(data_query, fast_weights) loss = criterion(logits, y) loss.backward(retain_graph=True) # Get post-update accuracies y_pred = logits.softmax(-1) acc = accuracy(y_pred, y)[0] # measure elapsed time recoder.tok() recoder.tik() recoder.data_tik() # update average value vals = [loss.item(),acc] recoder.update(vals) if i % log_interval == log_interval-1: recoder.log(epoch,i,len(valloader),mode='Eval') return recoder.get_avg('val acc')