def eval_gcr(model, criterion,
valloader, device, epoch,
log_interval, writer, args):
batch_time = AverageMeter()
data_time = AverageMeter()
avg_loss1 = AverageMeter()
avg_loss2 = AverageMeter()
avg_acc1 = AverageMeter()
avg_acc2 = AverageMeter()
statistic = []
# Create recorder
averagers = [avg_loss1, avg_loss2, avg_acc1, avg_acc2]
names = ['val loss1','val loss2','val acc1','val acc2']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
for i, batch in enumerate(valloader):
with torch.no_grad():
# measure data loading time
recoder.data_tok()
# get the inputs and labels
data, lab = [_.to(device) for _ in batch]
# forward
p = args.shot * args.test_way
data_shot = data[:p]
data_query = data[p:]
logits, label, logits2, gt = \
model(data_shot,data_query,lab,mode='eval')
# compute the loss
loss, loss1, loss2 = criterion(logits, label, logits2, gt)
# print('logits: {}'.format(logits))
# print('out: {}'.format(logits.argmax(-1)))
# print('label: {}'.format(label))
# compute the metrics
acc1 = accuracy(logits, label)[0]
acc2 = accuracy(logits2, gt)[0]
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value & account statistic
vals = [loss1.item(),loss2.item(),acc1,acc2]
recoder.update(vals)
statistic.append(acc1.data.cpu().numpy())
if i % log_interval == log_interval-1:
recoder.log(epoch,i,len(valloader),mode='Eval')
return recoder.get_avg('val acc1'), numpy.array(statistic)
def eval_mn_pn(model, criterion,
valloader, device, epoch,
log_interval, writer, args):
batch_time = AverageMeter()
data_time = AverageMeter()
avg_loss = AverageMeter()
avg_acc = AverageMeter()
statistic = []
# Create recorder
averagers = [avg_loss,avg_acc]
names = ['val loss','val acc']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
for i, batch in enumerate(valloader):
with torch.no_grad():
# measure data loading time
recoder.data_tok()
# get the inputs and labels
data, lab = [_.to(device) for _ in batch]
# forward
p = args.shot * args.test_way
data_shot = data[:p]
data_query = data[p:]
y_pred, label = model(data_shot,data_query,mode='eval')
# print('lab: {}'.format(lab.view((args.shot+args.query_val),args.test_way)[0]))
# compute the loss
loss = criterion(y_pred, label)
# print('y_pred: {}'.format(y_pred.argmax(-1)))
# print('label: {}'.format(label))
# compute the metrics
acc = accuracy(y_pred, label)[0]
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value & account statistic
vals = [loss.item(),acc]
recoder.update(vals)
statistic.append(acc.data.cpu().numpy())
if i % log_interval == log_interval-1:
recoder.log(epoch,i,len(valloader),mode='Eval')
return recoder.get_avg('val acc'), numpy.array(statistic)
def test_gcr(model, criterion,
valloader, device, epoch,
log_interval, writer, args, relation):
batch_time = AverageMeter()
data_time = AverageMeter()
avg_loss = AverageMeter()
avg_acc = AverageMeter()
# Create recorder
averagers = [avg_loss, avg_acc]
names = ['val loss', 'val acc']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
for i, batch in enumerate(valloader):
with torch.no_grad():
# measure data loading time
recoder.data_tok()
# get the inputs and labels
data, lab = [_.to(device) for _ in batch]
# forward
proto = model.baseModel(data)
global_set = torch.cat([model.global_base,model.global_novel])
logits = relation(proto,global_set)
# print('logits: ',logits.argmax(-1))
# print('lab: ',lab)
# compute the loss
loss = criterion(logits, lab)
# compute the metrics
acc = accuracy(logits, lab)[0]
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value
vals = [loss.item(),acc]
recoder.update(vals)
if i % log_interval == log_interval-1:
recoder.log(epoch,i,len(valloader),mode='Test')
return recoder.get_avg('val acc')
def eval_cnn(model, criterion, valloader,
device, epoch, log_interval, writer, args):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
avg_acc = AverageMeter()
averagers = [losses, avg_acc]
names = ['val loss','val acc']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
for i, batch in enumerate(valloader):
with torch.no_grad():
# measure data loading time
recoder.data_tok()
# get the data and labels
data,lab = [_.to(device) for _ in batch]
p = args.shot * args.test_way
data_shot = data[:p]
data_query = data[p:]
input = torch.cat([data_shot,data_query],0)
# forward
outputs = model(input)
# compute the loss
loss = criterion(outputs,lab)
# compute the metrics
acc = accuracy(outputs, lab)[0]
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value
vals = [loss.item(),acc]
recoder.update(vals)
# logging
if i==0 or i % log_interval == log_interval-1 or i==len(valloader)-1:
recoder.log(epoch,i,len(valloader),mode='Eval')
return recoder.get_avg('val acc')
def eval_c3d(model, criterion, valloader,
device, epoch, log_interval, writer, eval_samples):
batch_time = AverageMeter()
data_time = AverageMeter()
losses = AverageMeter()
avg_top1 = AverageMeter()
avg_top5 = AverageMeter()
averagers = [losses, avg_top1, avg_top5]
names = ['val loss','val top1','val top5']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
for i, batch in enumerate(valloader):
with torch.no_grad():
# Reduce the evaluation time !!!
if i>eval_samples: break
# measure data loading time
recoder.data_tok()
# get the data and labels
data,lab = [_.to(device) for _ in batch]
# forward
outputs = model(data)
# compute the loss
loss = criterion(outputs,lab)
# compute the metrics
top1, top5 = accuracy(outputs, lab, topk=(1,5))
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value
vals = [loss.item(),top1,top5]
recoder.update(vals)
# logging
if i==0 or i % log_interval == log_interval-1 or i==len(valloader)-1:
recoder.log(epoch,i,len(valloader),mode='Eval')
return recoder.get_avg('val top1')
def test_mn(model, global_proto, criterion,
valloader, device, epoch,
log_interval, writer, args):
batch_time = AverageMeter()
data_time = AverageMeter()
avg_loss = AverageMeter()
avg_acc = AverageMeter()
statistic = []
# Create recorder
averagers = [avg_loss,avg_acc]
names = ['val loss','val acc']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
for i, batch in enumerate(valloader):
with torch.no_grad():
# measure data loading time
recoder.data_tok()
# get the inputs and labels
data, lab = [_.to(device) for _ in batch]
# forward
y_pred = model.gfsl_test(global_proto,data)
# compute the loss
loss = criterion(y_pred, lab)
# compute the metrics
acc = accuracy(y_pred, lab)[0]
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value & account statistic
vals = [loss.item(),acc]
recoder.update(vals)
if i % log_interval == log_interval-1:
recoder.log(epoch,i,len(valloader),mode='Test')
return recoder.get_avg('val acc')
def test_seq2seq(model, criterion, dataloader, device, epoch, log_interval,
writer):
batch_time = AverageMeter()
data_time = AverageMeter()
avg_loss = AverageMeter()
avg_acc = AverageMeter()
avg_wer = AverageMeter()
# Create recorder
averagers = [avg_loss, avg_acc, avg_wer]
names = ['test loss', 'test acc', 'test wer']
recoder = Recorder(averagers, names, writer, batch_time, data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
with torch.no_grad():
for batch_idx, batch in enumerate(dataloader):
# measure data loading time
recoder.data_tok()
# get the data and labels
imgs = batch['videos'].cuda()
target = batch['annotations'].permute(1, 0).contiguous().cuda()
# forward(no teacher forcing)
outputs = model(imgs, target, 0)
# target: (batch_size, trg len)
# outputs: (trg_len, batch_size, output_dim)
# skip sos
output_dim = outputs.shape[-1]
outputs = outputs[1:].view(-1, output_dim)
target = target.permute(1, 0)[1:].reshape(-1)
# compute the loss
loss = criterion(outputs, target)
# compute the accuracy
prediction = torch.max(outputs, 1)[1]
score = accuracy_score(target.cpu().data.squeeze().numpy(),
prediction.cpu().data.squeeze().numpy())
# compute wer
# prediction: ((trg_len-1)*batch_size)
# target: ((trg_len-1)*batch_size)
batch_size = imgs.shape[0]
prediction = prediction.view(-1, batch_size).permute(1, 0).tolist()
target = target.view(-1, batch_size).permute(1, 0).tolist()
wers = []
for i in range(batch_size):
# add mask(remove padding, eos, sos)
prediction[i] = [
item for item in prediction[i] if item not in [0, 1, 2]
]
target[i] = [
item for item in target[i] if item not in [0, 1, 2]
]
wers.append(wer(target[i], prediction[i]))
batch_wer = sum(wers) / len(wers)
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value
vals = [loss.item(), score, batch_wer]
b = imgs.size(0)
recoder.update(vals, count=b)
# logging
if batch_idx == 0 or batch_idx % log_interval == log_interval - 1 or batch_idx == len(
dataloader) - 1:
recoder.log(epoch, batch_idx, len(dataloader), mode='Test')
return recoder.get_avg('val acc')
def evaluate_confusion_matrix(model, criterion,
valloader, device, epoch,
log_interval, writer, args, relation, name,
category_space='novel',
base_class=400,novel_class=100):
batch_time = AverageMeter()
data_time = AverageMeter()
avg_loss = AverageMeter()
avg_acc = AverageMeter()
# Create recorder
averagers = [avg_loss, avg_acc]
names = ['val loss', 'val acc']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
num_class = base_class + novel_class
cmat = numpy.zeros([num_class,num_class])
for i, batch in enumerate(valloader):
with torch.no_grad():
# measure data loading time
recoder.data_tok()
# get the inputs and labels
data, lab = [_.to(device) for _ in batch]
# forward
proto = model.baseModel(data)
global_set = torch.cat([model.global_base,model.global_novel])
logits = relation(proto,global_set)
# print('logits: ',logits.argmax(-1))
# print('lab: ',lab)
# compute the loss
loss = criterion(logits, lab)
# compute the metrics
acc = accuracy(logits, lab)[0]
# compute the confusion matrix
predict = logits.argmax(-1)
for p,g in zip(predict,lab):
cmat[g,p] += 1
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value
vals = [loss.item(),acc]
recoder.update(vals)
if i % log_interval == log_interval-1:
recoder.log(epoch,i,len(valloader),mode='Test')
# normalize & print confusion matrix
cmat = cmat / cmat.sum(1)
if category_space == 'novel':
cmat = cmat[base_class:,base_class:]
elif category_space == 'base':
cmat = cmat[:base_class,:base_class]
df = pd.DataFrame(cmat)
df.to_csv(name)
return recoder.get_avg('val acc')
def eval_maml(model, criterion,
valloader, device, epoch,
log_interval, writer, args):
batch_time = AverageMeter()
data_time = AverageMeter()
avg_loss = AverageMeter()
avg_acc = AverageMeter()
# Create recorder
averagers = [avg_loss, avg_acc]
names = ['val loss','val acc']
recoder = Recorder(averagers,names,writer,batch_time,data_time)
# Set evaluation mode
model.eval()
recoder.tik()
recoder.data_tik()
# Settings
create_graph = (True if args.order == 2 else False)
for i, batch in enumerate(valloader):
# measure data loading time
recoder.data_tok()
# get the inputs and labels
data, lab = [_.to(device) for _ in batch]
# forward
# data = data.view( ((args.shot+args.query),args.train_way) + data.size()[-3:] )
# data = data.permute(1,0,2,3,4).contiguous()
# data = data.view( (-1,) + data.size()[-3:] )
p = args.shot * args.test_way
data_shot = data[:p]
data_query = data[p:]
data_shape = data.size()[-3:]
# Create a fast model using the current meta model weights
fast_weights = OrderedDict(model.named_parameters())
# Train the model for `inner_train_steps` iterations
for inner_batch in range(args.inner_train_steps):
# Perform update of model weights
y = create_nshot_task_label(args.test_way, args.shot).to(device)
logits = model.functional_forward(data_shot, fast_weights)
loss = criterion(logits, y)
gradients = torch.autograd.grad(loss, fast_weights.values(), create_graph=create_graph)
# Update weights manually
fast_weights = OrderedDict(
(name, param - args.inner_lr * grad)
for ((name, param), grad) in zip(fast_weights.items(), gradients)
)
# Do a pass of the model on the validation data from the current task
y = create_nshot_task_label(args.test_way, args.query_val).to(device)
logits = model.functional_forward(data_query, fast_weights)
loss = criterion(logits, y)
loss.backward(retain_graph=True)
# Get post-update accuracies
y_pred = logits.softmax(-1)
acc = accuracy(y_pred, y)[0]
# measure elapsed time
recoder.tok()
recoder.tik()
recoder.data_tik()
# update average value
vals = [loss.item(),acc]
recoder.update(vals)
if i % log_interval == log_interval-1:
recoder.log(epoch,i,len(valloader),mode='Eval')
return recoder.get_avg('val acc')