def test(model, quesfeaShu, labelShu, lengthShu):
model.eval()
idx = sorted(range(len(lengthShu)), key=lambda x: lengthShu[x], reverse=True)
_quesfeaShu = []
_labelShu = []
_lengthShu = []
for j in range(len(idx)):
_quesfeaShu.append(quesfeaShu[idx[j]])
_labelShu.append(labelShu[idx[j]])
_lengthShu.append(lengthShu[idx[j]])
questrainarray = np.asarray(_quesfeaShu)
labeltrainarray = np.asarray(_labelShu)
lengthtrainarray = np.asarray(_lengthShu)
tmp = [questrainarray, labeltrainarray, lengthtrainarray]
tmp = [Variable(torch.from_numpy(_), requires_grad=False) for _ in tmp]
trques, trlabel, length = tmp
if args.cuda:
trlabel.cuda()
output = model(trques, length)
# st(context=27)
print("precesion 1 : %s" % accuracy(output.data, trlabel.data, topk=(1,), ori_label=labeltrainarray))
def export_onnx(path, batch_size, seq_len):
print('The model is also exported in ONNX format at {}'.
format(os.path.realpath(args.onnx_export)))
model.eval()
dummy_input = torch.LongTensor(seq_len * batch_size).zero_().view(-1, batch_size).to(device)
hidden = model.init_hidden(batch_size)
torch.onnx.export(model, (dummy_input, hidden), path)
def evaluate(data_source, batch_size=10, window=args.window):
# Turn on evaluation mode which disables dropout.
if args.model == 'QRNN': model.reset()
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
next_word_history = None
pointer_history = None
for i in range(0, data_source.size(0) - 1, args.bptt):
if i > 0: print(i, len(data_source), math.exp(total_loss / i))
data, targets = get_batch(data_source, i, evaluation=True, args=args)
output, hidden, rnn_outs, _ = model(data, hidden, return_h=True)
rnn_out = rnn_outs[-1].squeeze()
output_flat = output.view(-1, ntokens)
###
# Fill pointer history
start_idx = len(next_word_history) if next_word_history is not None else 0
next_word_history = torch.cat([one_hot(t.data[0], ntokens) for t in targets]) if next_word_history is None else torch.cat([next_word_history, torch.cat([one_hot(t.data[0], ntokens) for t in targets])])
#print(next_word_history)
pointer_history = Variable(rnn_out.data) if pointer_history is None else torch.cat([pointer_history, Variable(rnn_out.data)], dim=0)
#print(pointer_history)
###
# Built-in cross entropy
# total_loss += len(data) * criterion(output_flat, targets).data[0]
###
# Manual cross entropy
# softmax_output_flat = torch.nn.functional.softmax(output_flat)
# soft = torch.gather(softmax_output_flat, dim=1, index=targets.view(-1, 1))
# entropy = -torch.log(soft)
# total_loss += len(data) * entropy.mean().data[0]
###
# Pointer manual cross entropy
loss = 0
softmax_output_flat = torch.nn.functional.softmax(output_flat)
for idx, vocab_loss in enumerate(softmax_output_flat):
p = vocab_loss
if start_idx + idx > window:
valid_next_word = next_word_history[start_idx + idx - window:start_idx + idx]
valid_pointer_history = pointer_history[start_idx + idx - window:start_idx + idx]
logits = torch.mv(valid_pointer_history, rnn_out[idx])
theta = args.theta
ptr_attn = torch.nn.functional.softmax(theta * logits).view(-1, 1)
ptr_dist = (ptr_attn.expand_as(valid_next_word) * valid_next_word).sum(0).squeeze()
lambdah = args.lambdasm
p = lambdah * ptr_dist + (1 - lambdah) * vocab_loss
###
target_loss = p[targets[idx].data]
loss += (-torch.log(target_loss)).data[0]
total_loss += loss / batch_size
###
hidden = repackage_hidden(hidden)
next_word_history = next_word_history[-window:]
pointer_history = pointer_history[-window:]
return total_loss / len(data_source)
def valid(epoch, quesfeaShu, labelShu, lengthShu):
losses = AverageMeter()
top1 = AverageMeter()
model.eval()
start_time = time.time()
for i in range(0, len(quesfeaShu) / args.batch_size):
if i == len(quesfeaShu) / args.batch_size - 1:
batchend = len(quesfeaShu)
else:
batchend = (i + 1) * (args.batch_size)
# print batchend
batchstart = i * (args.batch_size)
batch_size = batchend - batchstart
quesfeabatch = []
labelbatch = []
lengthbatch = []
quesfeaOri = quesfeaShu[batchstart:batchend]
labelOri = labelShu[batchstart:batchend]
lengthOri = lengthShu[batchstart:batchend]
idxbatch = sorted(range(len(lengthOri)), key=lambda x: lengthOri[x], reverse=True)
for j in range(len(idxbatch)):
quesfeabatch.append(quesfeaOri[idxbatch[j]])
labelbatch.append(labelOri[idxbatch[j]])
lengthbatch.append(lengthOri[idxbatch[j]])
questrainarray = np.asarray(quesfeabatch)
labeltrainarray = np.asarray(labelbatch)
lengthtrainarray = np.asarray(lengthbatch)
tmp = [questrainarray, labeltrainarray, lengthtrainarray]
tmp = [Variable(torch.from_numpy(_), requires_grad=False) for _ in tmp]
trques, trlabel, length = tmp
if args.cuda:
trlabel.cuda()
output = model(trques, length)
# print output
loss = criterion(output, trlabel) / (batch_size)
prec1, = accuracy(output.data, trlabel.data, topk=(1,), ori_label=labeltrainarray)
# label 0 or 1
losses.update(loss.data[0], batch_size)
top1.update(prec1[0], batch_size)
# loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
torch.nn.utils.clip_grad_norm(model.parameters(), args.clip)
print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'batch_valid ' + str(i)
# update better performance model
global best_score
if top1.avg > best_score:
torch.save(model, args.save)
print 'save model'
best_score = top1.avg
print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'epoch_valid ' + str(epoch)
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
if args.model == 'QRNN': model.reset()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args, evaluation=True)
output, hidden = model(data, hidden)
total_loss += len(data) * criterion(model.decoder.weight, model.decoder.bias, output, targets).data
hidden = repackage_hidden(hidden)
return total_loss.item() / len(data_source)
def evaluate(data_source):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(eval_batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, evaluation=True)
output, hidden = model(data, hidden)
output_flat = output.view(-1, ntokens)
total_loss += len(data) * criterion(output_flat, targets).data
hidden = repackage_hidden(hidden)
return total_loss[0] / len(data_source)
def valid(epoch, quesfeaShu, labelShu, lengthShu):
top1 = AverageMeter()
model.eval()
for i in range(0, len(quesfeaShu) / args.batch_size):
if i == len(quesfeaShu) / args.batch_size - 1:
batchend = len(quesfeaShu)
else:
batchend = (i + 1) * (args.batch_size)
# print batchend
batchstart = i * (args.batch_size)
batch_size = batchend - batchstart
quesfeabatch = []
labelbatch = []
lengthbatch = []
quesfeaOri = quesfeaShu[batchstart:batchend]
labelOri = labelShu[batchstart:batchend]
lengthOri = lengthShu[batchstart:batchend]
idxbatch = sorted(range(len(lengthOri)), key=lambda x: lengthOri[x], reverse=True)
for j in range(len(idxbatch)):
quesfeabatch.append(quesfeaOri[idxbatch[j]])
labelbatch.append(labelOri[idxbatch[j]])
lengthbatch.append(lengthOri[idxbatch[j]])
questrainarray = np.asarray(quesfeabatch)
labeltrainarray = np.asarray(labelbatch)
lengthtrainarray = np.asarray(lengthbatch)
tmp = [questrainarray, labeltrainarray, lengthtrainarray]
tmp = [Variable(torch.from_numpy(_), requires_grad=False) for _ in tmp]
trques, trlabel, length = tmp
if args.cuda:
trlabel.cuda()
output = model(trques, length)
loss = criterion(output, trlabel) / (batch_size)
prec1, = accuracy(output.data, trlabel.data, topk=(1,))
top1.update(prec1[0], batch_size)
print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'batch_valid ' + str(i)
global best_score
if top1.avg > best_score:
torch.save(model, args.save)
print 'save model'
best_score = top1.avg
print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'epoch_valid ' + str(epoch)
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args, evaluation=True)
targets = targets.view(-1)
log_prob, hidden = parallel_model(data, hidden)
loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)), targets).data
total_loss += len(data) * loss
hidden = repackage_hidden(hidden)
return total_loss[0] / len(data_source)
def test_model(loader, model):
"""
Help function that tests the models's performance on a dataset
:param: loader: data loader for the dataset to test against
"""
correct = 0
total = 0
model.eval()
for data, labels in loader:
if args.cuda:
data, labels = data.cuda(), labels.cuda()
data = Variable(data)
outputs = model(data)
predicted = (outputs.max(1)[1].data.long()).view(-1)
total += labels.size(0)
correct += (predicted == labels).sum()
model.train()
return 100 * correct / total
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
with torch.no_grad():
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args)
targets = targets.view(-1)
log_prob, hidden = parallel_model(data, hidden)
loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)),
targets).data
total_loss += len(data) * loss
hidden = repackage_hidden(hidden)
return total_loss.item() / len(data_source)
def evaluate(data_source):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0
hidden = model.init_hidden(eval_batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args.bptt, evaluation=True)
#> output has size seq_length x batch_size x vocab_size
output, hidden = model(data, None)
#> output_flat has size num_targets x vocab_size (batches are stacked together)
#> ! important, otherwise softmax computation (e.g. with F.softmax()) is incorrect
output_flat = output.view(-1, ntokens)
#output_candidates_info(output_flat.data, targets.data)
total_loss += len(data) * nn.CrossEntropyLoss()(output_flat, targets).data
# hidden = repackage_hidden(hidden)
return total_loss[0] /len(data_source)
def evaluate(data_source):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0.
ntokens = len(corpus.dictionary)
if args.model != 'Transformer':
hidden = model.init_hidden(eval_batch_size)
with torch.no_grad():
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i)
if args.model == 'Transformer':
output = model(data)
else:
output, hidden = model(data, hidden)
hidden = repackage_hidden(hidden)
output_flat = output.view(-1, ntokens)
total_loss += len(data) * criterion(output_flat, targets).item()
return total_loss / (len(data_source) - 1)
def evaluate():
running_loss_eval = 0.0
model.eval()
for i, (input, mask, name) in enumerate(val_loader):
input, mask = input[0].to(device=device)[None,:,:,:], mask[0].to(device=device)[None,:,:,:]
with torch.no_grad():
output = model(input)
# torch.where(outputs > 0.5, torch.ones(1).cuda(),torch.zeros(1).cuda())
loss = criterion(output, mask)
loss_item = loss.item()
running_loss_eval += loss_item
print(f"Eval: {epoch}, iteration: {i} of {len(val_loader)}, loss: {loss_item}")
eval_loss.append(running_loss_eval / len(val_loader))
def evaluate(data_source):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
if (not args.single) and (torch.cuda.device_count() > 1):
#"module" is necessary when using DataParallel
hidden = model.module.init_hidden(eval_batch_size)
else:
hidden = model.init_hidden(eval_batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, evaluation=True)
output, hidden = model(data, hidden)
output_flat = output.view(-1, ntokens)
curr_loss = len(data) * criterion(output_flat, targets).data
total_loss += curr_loss
hidden = repackage_hidden(hidden)
return total_loss[0] / len(data_source)
def evaluate(source):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0.
if args.model != 'Transformer':
hidden = model.init_hidden(eval_batch_size)
# Speed up evaluation with torch.no_grad()
with torch.no_grad():
for i in range(0, source.size(0) - 1, args.seq_len):
data, targets = get_batch(source, i)
if args.model == 'Transformer':
output = model(data)
else:
output, hidden = model(data, hidden)
hidden = repackage_hidden(hidden)
loss = criterion(output.view(-1, ntokens), targets)
total_loss += len(data) * loss.item()
return total_loss / (len(source) - 1)
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
if args.model == 'QRNN': model.reset()
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args, evaluation=True)
output, hidden = model(data, hidden)
if isinstance(criterion, SplitCrossEntropyLoss):
total_loss += len(data) * criterion(
model.decoder.weight, model.decoder.bias, output, targets).data
else:
output_flat = output.view(-1, ntokens)
total_loss += len(data) * criterion(output_flat, targets).data
hidden = repackage_hidden(hidden)
return total_loss.item() / len(data_source)
def train_step(input_data,
y,
optimizer,
criterion,
dev_data,
batch_size=config.batch_size):
model.train()
count = 0
total_loss = 0
for i in range(0, len(y), batch_size):
optimizer.zero_grad()
loss = 0
'''
word_input = input_data[0]
pos_input = input_data[1]
label_input = input_data[2]
'''
input_batch, y_batch = get_batch(input_data, y, i, batch_size)
print("run minibatch :%d " % i)
#input_batch,y_batch=get_batch(input_data,y,i,batch_size)
y_batch = torch.from_numpy(y_batch.nonzero()[1]).long().to(device)
y_predict_logits = model(input_batch)
loss = criterion(y_predict_logits, y_batch)
loss.backward()
optimizer.step()
total_loss += loss.item()
print("minibatch : %d ,loss: %.5f" % (i, loss.item()))
count += 1
print("-----------------------------------------------")
print("avg loss : %.5f" % (total_loss / count))
print("-----------------------------------------------")
print("Evauating on dev set...")
model.eval()
UAS = dev_step(dev_data)
print("dev UAS :%.3f" % UAS)
return UAS
def F1_matrix_score_for_data(model, A, data_loader, batch_size, top_k):
device = model.device
nb_batch = len(data_loader.dataset) // batch_size
if len(data_loader.dataset) % batch_size == 0:
total_batch = nb_batch
else:
total_batch = nb_batch + 1
print("Total batch: ", total_batch)
list_R_score = []
list_P_score = []
list_F1_score = []
model.eval()
with torch.no_grad():
for i, data_pack in enumerate(data_loader, 0):
data_x, data_seq_len, data_y = data_pack
x_ = data_x.to_dense().to(dtype=model.dtype, device=device)
real_batch_size = x_.size()[0]
# hidden = model.init_hidden(real_batch_size)
y_ = data_y.to(dtype=model.dtype, device=device)
logits_ = model(A, data_seq_len, x_)
predict_basket = utils.predict_top_k(logits_, top_k,
real_batch_size,
model.nb_items)
target_basket_np = y_.cpu().numpy()
correct_predict = predict_basket * target_basket_np
nb_correct = np.count_nonzero(correct_predict, axis=1)
actual_basket_size = np.count_nonzero(target_basket_np, axis=1)
batch_recall = nb_correct / actual_basket_size
batch_precision = nb_correct / top_k
batch_f1 = np.zeros_like(nb_correct, dtype=float)
for i in range(len(batch_f1)):
if (nb_correct[i] > 0):
batch_f1[i] = (2 *
(batch_precision[i] * batch_recall[i])) / (
batch_precision[i] + batch_recall[i])
list_P_score.append(batch_precision[i])
list_R_score.append(batch_recall[i])
list_F1_score.append(batch_f1[i])
# print(list_MRR_score)
# print("MRR score: %.6f" % np.array(list_MRR_score).mean())
return np.array(list_R_score).mean(), np.array(
list_P_score).mean(), np.array(list_F1_score).mean()
def train_loop(model, loss_function, optimizer, scheduler, args):
training_loss = []
train_ldr = train_dataloader(args)
validation_loss = []
test_ldr = test_dataloader(args)
for epoch in range(1, args.epochs + 1):
## Training
train_loss = 0
for batch_idx, batch in enumerate(train_ldr):
# Sets the model into training mode
model.train()
loss = training_step(batch, batch_idx, model, loss_function, optimizer, args.device, train_on_y2=args.train_on_y2)
train_loss += loss.item()
if batch_idx % args.log_interval == 0:
x,y, y2=batch
print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.2e}'.format(epoch,
batch_idx * len(x), len(train_ldr.dataset),100. * batch_idx / len(train_ldr),
loss.item() / len(x)))
average_loss = train_loss / len(train_ldr.dataset)
print('====> Epoch: {} Average loss: {:.2e}'.format(epoch, average_loss))
training_loss.append(average_loss)
scheduler.step()
## Testing
test_loss = 0
for batch_idx, batch in enumerate(test_ldr):
model.eval()
loss = validation_step(batch, batch_idx, model, loss_function, args.device, train_on_y2=args.train_on_y2)
test_loss += loss.item()
average_loss_val = test_loss / len(test_ldr.dataset)
print('====> validation loss: {:.2e}'.format(average_loss_val))
validation_loss.append(average_loss_val)
if epoch % 100 == 0:
checkpoint_save(epoch, model, optimizer, training_loss, validation_loss, args.model_name, args.locations, args)
# Save the final model
torch.save({'epoch':epoch,
'model_state_dict':model.state_dict(),
'optimizer_state_dict':optimizer.state_dict(),
'training_loss':training_loss,
'validation_loss':validation_loss,
'arguments':args},
args.locations['model_loc']+'/'+args.model_name)
return training_loss, validation_loss
def evaluate(data_source):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0.
ntokens = len(corpus.dictionary)
# 2*10*200
hidden = model.init_hidden(eval_batch_size)
with torch.no_grad():
for i in range(0, data_source.size(0) - 1, args.bptt):
# data:35*10, targets:350
data, targets = get_batch(data_source, i)
# 参数：(35*10, 2*10*200), output=(35,10,33278) hidden=(2,10,200)
output, hidden = model(data, hidden)
# output_flat == > 350*33278
output_flat = output.view(-1, ntokens)
# len(data)=35, 称为sequence length
total_loss += len(data) * criterion(output_flat, targets).item()
hidden = repackage_hidden(hidden)
return total_loss / (len(data_source) - 1)
def test(model, perm=torch.arange(0, 224 * 224 * 3).long()):
model.eval()
test_loss = 0
correct = 0
for data, target in test_loader:
output = model(data)
test_loss += F.nll_loss(output, target,
reduction='sum').item() # sum up batch loss
pred = output.data.max(
1, keepdim=True)[1] # get the index of the max log-probability
correct += pred.eq(target.data.view_as(pred)).cpu().sum().item()
test_loss /= len(test_loader.dataset)
accuracy = 100. * correct / len(test_loader.dataset)
accuracy_list.append(accuracy)
print(
'\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
test_loss, correct, len(test_loader.dataset), accuracy))
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
if args.model == 'QRNN': model.reset()
total_loss = 0
ntokens = len(corpus['words'].idx2word)
all_outputs = []
for i in range(0, len(data_source['sentences']) - 1, batch_size):
data, lengths, max_length, targets = get_batch(data_source, i, batch_size)
cur_batch_size = data.size(1)
hidden = model.init_hidden(cur_batch_size)
output, hidden = model(data, lengths, max_length, hidden)
for p in output.tolist():
all_outputs.append(p)
loss = batch_size * criterion(output, targets.long())
total_loss += loss
hidden = repackage_hidden(hidden)
# return total_loss.item() / batch_size
return total_loss.item() / len(data_source['sentences']), all_outputs
def evaluate(split):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss, nbatches = 0, 0
ntokens = len(corpus.dictionary.idx2word) #2740
bsz = 64
hidden = model.init_hidden(bsz)
for source, target in corpus.tokenize('./Data/Dev/dev.csv'):
#print("Matrices")
#print(hidden.shape)
#print(source.shape)
print(source.size())
print(hidden.size())
output, hidden = model(source, hidden)
output_flat = output.view(-1, ntokens)
total_loss += criterion(output_flat, target.view(-1)).data
hidden = repackage_hidden(hidden)
nbatches += 1
return total_loss[0] / nbatches
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
if args.model == 'QRNN':
model.reset()
loss_measure = AverageMeter()
acc_measure = AverageMeter()
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args, evaluation=True)
output, hidden = model(data, hidden)
loss = criterion(model.decoder.weight, model.decoder.bias, output,
targets).data
loss_measure.update(float(loss), targets.nelement())
acc = float(accuracy(output.data, targets.data)[0])
acc_measure.update(acc, targets.nelement())
hidden = repackage_hidden(hidden)
return loss_measure.avg, acc_measure.avg
def validate(val_loader, model, criterion):
batch_time = AverageMeter()
losses = AverageMeter()
acc = AverageMeter()
# switch to evaluate mode
model.eval()
end = time.time()
for i, (images, labels) in enumerate(val_loader):
image_var = torch.tensor(images).cuda(async=True)
target = torch.tensor(labels).cuda(async=True)
# 图片前传。验证和测试时不需要更新网络权重，所以使用torch.no_grad()，表示不计算梯度
with torch.no_grad():
y_pred = model(image_var)
loss = criterion(y_pred, target)
# measure accuracy and record loss
prec, PRED_COUNT = accuracy(y_pred.data, labels, topk=(1, 1))
losses.update(loss.item(), images.size(0))
acc.update(prec, PRED_COUNT)
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % print_freq == 0:
print('TrainVal: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Accuray {acc.val:.3f} ({acc.avg:.3f})'.format(
i,
len(val_loader),
batch_time=batch_time,
loss=losses,
acc=acc))
print(' * Accuray {acc.avg:.3f}'.format(acc=acc),
'(Previous Best Acc: %.3f)' % best_precision,
' * Loss {loss.avg:.3f}'.format(loss=losses),
'Previous Lowest Loss: %.3f)' % lowest_loss)
return acc.avg, losses.avg
def check_accuracy(model: nn.Module, data_loader: DataLoader):
"""
if loader.dataset.train:
print('Checking accuracy on validation set')
else:
print('Checking accuracy on test set')
"""
num_correct = 0
num_samples = 0
model.eval() # set model to evaluation mode
with torch.no_grad():
for data, target, _ in data_loader:
data, target = data.to(device=DEVICE), target.to(device=DEVICE)
_, label, _ = model(data)
num_correct += (label == target).cpu().int().item()
num_samples += label.size(0)
acc = float(num_correct) / num_samples
print('Got {} / {} correct {:.2f} %'.format(num_correct, num_samples, 100 * acc))
return acc
def valid_model():
model.eval()
running_loss = 0.0
running_corrects = 0
for img_inputs, labels, paths in valid_loader:
img_inputs = img_inputs.to(device)
labels = labels.to(device)
tag_inputs = tag_generator(paths, img_tags, w2v).to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(False):
outputs = model(img_inputs, tag_inputs)
loss = criterion(outputs, labels)
_, preds = torch.max(outputs, 1)
running_loss += loss.item() * img_inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / len(valid_loader.dataset)
epoch_acc = running_corrects.double() / len(valid_loader.dataset)
print('valid Loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))
return epoch_loss, epoch_acc
def test_model():
model.load_state_dict(torch.load(args.tag+'_backup/checkpoint.pt'))
model.eval()
running_loss = 0.0
running_corrects = 0
for img_inputs, labels, paths in test_loader:
img_inputs = img_inputs.to(device)
labels = labels.to(device)
tag_inputs = tag_generator(paths, img_tags, w2v).to(device)
optimizer.zero_grad()
with torch.set_grad_enabled(False):
outputs = model(img_inputs, tag_inputs)
loss = criterion(outputs, labels)
_, preds = torch.max(outputs, 1)
running_loss += loss.item() * img_inputs.size(0)
running_corrects += torch.sum(preds == labels.data)
epoch_loss = running_loss / len(test_loader.dataset)
epoch_acc = running_corrects.double() / len(test_loader.dataset)
print('test Loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))
def evaluate(model, criterion, data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
model_now = model.module
criterion_now = criterion.module
if args.model == 'QRNN': model_now.reset()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model_now.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args, evaluation=True)
output, hidden = model_now(data, hidden)
criterion_now.replicate_weight_and_bias(
torch.nn.Parameter(model.module.decoder.weight),
torch.nn.Parameter(model.module.decoder.bias))
total_loss += len(data) * criterion_now(hiddens=output,
targets=targets).data
hidden = repackage_hidden(hidden)
return total_loss.item() / len(data_source)
def evaluate(data_source):
# Turn on evaluation mode which disables dropout.
model.eval()
eval_criterion = nn.CrossEntropyLoss()
total_loss = 0.
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(eval_batch_size)
with torch.no_grad():
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i)
# gs534 add sentence resetting
eosidx = corpus.dictionary.get_eos()
output, hidden = model(data, hidden, eosidx)
output_flat = output.view(-1, ntokens)
total_loss += len(data) * eval_criterion(output_flat,
targets).item()
hidden = repackage_hidden(hidden)
return total_loss / len(data_source)
def evaluate(data_source):
# Turn on evaluation mode which disables dropout.
with torch.no_grad():
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(
eval_batch_size) #hidden size(nlayers, bsz, hdsize)
for i in range(0,
data_source.size(0) - 1,
args.bptt): # iterate over every timestep
data, targets = get_batch(data_source, i)
output, hidden = model(data, hidden)
# model input and output
# inputdata size(bptt, bsz), and size(bptt, bsz, embsize) after embedding
# output size(bptt*bsz, ntoken)
total_loss += len(data) * criterion(output, targets).data
hidden = repackage_hidden(hidden)
return total_loss / len(data_source)
def MRR_score_for_data(model, A, data_loader, batch_size):
device = model.device
nb_batch = len(data_loader.dataset) // batch_size
if len(data_loader.dataset) % batch_size == 0:
total_batch = nb_batch
else:
total_batch = nb_batch + 1
print("Total batch: ", total_batch)
list_MRR_score = []
model.eval()
with torch.no_grad():
for i, data_pack in enumerate(data_loader, 0):
data_x, data_seq_len, data_y = data_pack
x_ = data_x.to_dense().to(dtype=model.dtype, device=device)
real_batch_size = x_.size()[0]
# hidden = model.init_hidden(real_batch_size)
y_ = data_y.to(dtype=model.dtype, device=device)
predict_ = model(A, data_seq_len, x_)
sigmoid_pred = torch.sigmoid(predict_)
sorted_rank, indices = torch.sort(sigmoid_pred, descending=True)
for seq_idx, a_seq_idx in enumerate(y_):
# print(seq_idx)
idx_item_in_target_basket = (a_seq_idx == 1.0).nonzero(
as_tuple=True)[0]
# print(idx_item_in_target_basket)
sum_of_rank_score = 0
for idx_item in idx_item_in_target_basket:
item_rank = (indices[seq_idx] == idx_item).nonzero(
as_tuple=True)[0].item()
# print("Rank %d" % (item_rank + 1))
rank_score = 1 / (item_rank + 1)
sum_of_rank_score += rank_score
# print("sum of rank item in target: %.6f" % sum_of_rank_score)
target_basket_size = idx_item_in_target_basket.size()[0]
MRR_score = sum_of_rank_score / target_basket_size
# print(MRR_score)
list_MRR_score.append(MRR_score)
# print(list_MRR_score)
# print("MRR score: %.6f" % np.array(list_MRR_score).mean())
# print("MRR list len: %d" % len(list_MRR_score))
return np.array(list_MRR_score).mean()
def evaluate_model(model, val_loader, epoch, num_epochs, writer, current_lr,
log_every=20):
n_classes = model.n_classes
metric = torch.nn.CrossEntropyLoss()
model.eval()
for m in model.modules():
if isinstance(m, nn.BatchNorm2d):
m.train()
m.weight.requires_grad = False
m.bias.requires_grad = False
y_probs = np.zeros((0, n_classes), np.float)
losses, y_trues = [], []
for i, (image, label, case_id) in enumerate(val_loader):
# if torch.cuda.is_available():
# image = image.cuda()
# label = label.cuda()
prediction = model.forward(image.float())
loss = metric(prediction, label.long())
loss_value = loss.item()
losses.append(loss_value)
y_prob = F.softmax(prediction, dim=1)
y_probs = np.concatenate([y_probs, y_prob.detach().cpu().numpy()])
y_trues.append(label.item())
metric_collects = utils.calc_multi_cls_measures(y_probs, y_trues)
n_iter = epoch * len(val_loader) + i
writer.add_scalar('Val/Loss', loss_value, n_iter)
if (i % log_every == 0) & (i > 0):
prefix = '*Val|'
utils.print_progress(epoch + 1, num_epochs, i, len(val_loader),
np.mean(losses), current_lr, metric_collects,
prefix=prefix)
val_loss_epoch = np.round(np.mean(losses), 4)
return val_loss_epoch, metric_collects
def evaluate(model, data_source, batch_size=10):
model.eval()
if args.model == 'QRNN':
model.reset()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.seq_len):
data, targets = get_batch(data_source, i, args, evaluation=True)
output = model(data, hidden)
if isinstance(output, tuple):
output, hidden = output
output_flat = output.view(-1, ntokens)
total_loss += len(data) * criterion(output_flat, targets).data
hidden = repackage_hidden(hidden)
return total_loss[0] / len(data_source)
def evaluate(data_source, source_sampler, target_sampler, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
if args.model == 'QRNN':
model.reset()
total_loss = 0
hidden = model.init_hidden(batch_size)
for source_sample, target_sample in zip(source_sampler, target_sampler):
model.train()
data = Variable(torch.stack([data_source[i] for i in source_sample]),
volatile=True)
targets = Variable(torch.stack([data_source[i]
for i in target_sample])).view(-1)
output, hidden = model(data, hidden)
total_loss += len(data) * criterion(
model.decoder.weight, model.decoder.bias, output, targets).data
hidden = repackage_hidden(hidden)
return total_loss[0] / len(data_source)
def evaluate(input1, input2, label):
model.eval()
output = model(input1)
_, prediction = torch.max(output.data, 1)
label = label.data
count = 0
count0 = 0
count1 = 0
count2 = 0
for i in range(len(label)):
if label[i] == prediction[i]:
count += 1
return count / len(label)
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
if args.model == 'QRNN': model.reset()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
data, targets = get_batch(data_source, i, args, evaluation=True)
output, hidden = model(data, hidden)
input = torch.mm(model.decoder.weight, output.transpose(
0, 1)).transpose(0, 1) + model.decoder.bias
if args.loss == 'splitcrossentropy':
total_loss += len(data) * criterion(
model.decoder.weight, model.decoder.bias, output, targets).data
elif args.loss == 'focal':
total_loss += len(data) * criterion(input, targets, test=True).data
hidden = repackage_hidden(hidden)
return total_loss.item() / len(data_source)
def evaluate(data_source, batch_size=10):
# Turn on evaluation mode which disables dropout.
model.eval()
total_loss = 0
ntokens = len(corpus.dictionary)
hidden = model.init_hidden(batch_size)
for i in range(0, data_source.size(0) - 1, args.bptt):
print(i, data_source.size(0) - 1)
data, targets = get_batch(data_source, i, args, evaluation=True)
targets = targets.view(-1)
log_prob, hidden = parallel_model(data, hidden)
loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)),
targets).data
total_loss += loss * len(data)
hidden = repackage_hidden(hidden)
return total_loss[0] / len(data_source)
correct = pred.cpu().eq(indx_target).sum()
acc = correct * 1.0 / len(data)
print('Train Epoch: {} [{}/{}] Loss: {:.6f} Acc: {:.4f} lr: {:.2e}'.format(
epoch, batch_idx * len(data), len(train_loader.dataset),
loss.data[0], acc, optimizer.param_groups[0]['lr']))
elapse_time = time.time() - t_begin
speed_epoch = elapse_time / (epoch + 1)
speed_batch = speed_epoch / len(train_loader)
eta = speed_epoch * args.epochs - elapse_time
print("Elapsed {:.2f}s, {:.2f} s/epoch, {:.2f} s/batch, ets {:.2f}s".format(
elapse_time, speed_epoch, speed_batch, eta))
misc.model_snapshot(model, os.path.join(args.logdir, 'latest.pth'))
if epoch % args.test_interval == 0:
model.eval()
test_loss = 0
correct = 0
for data, target in test_loader:
indx_target = target.clone()
if args.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = model(data)
test_loss += F.cross_entropy(output, target).data[0]
pred = output.data.max(1)[1] # get the index of the max log-probability
correct += pred.cpu().eq(indx_target).sum()
test_loss = test_loss / len(test_loader) # average over number of mini-batch
acc = 100. * correct / len(test_loader.dataset)
print('\tTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(
