def test(model, quesfeaShu, labelShu, lengthShu): model.eval() idx = sorted(range(len(lengthShu)), key=lambda x: lengthShu[x], reverse=True) _quesfeaShu = [] _labelShu = [] _lengthShu = [] for j in range(len(idx)): _quesfeaShu.append(quesfeaShu[idx[j]]) _labelShu.append(labelShu[idx[j]]) _lengthShu.append(lengthShu[idx[j]]) questrainarray = np.asarray(_quesfeaShu) labeltrainarray = np.asarray(_labelShu) lengthtrainarray = np.asarray(_lengthShu) tmp = [questrainarray, labeltrainarray, lengthtrainarray] tmp = [Variable(torch.from_numpy(_), requires_grad=False) for _ in tmp] trques, trlabel, length = tmp if args.cuda: trlabel.cuda() output = model(trques, length) # st(context=27) print("precesion 1 : %s" % accuracy(output.data, trlabel.data, topk=(1,), ori_label=labeltrainarray))
def export_onnx(path, batch_size, seq_len): print('The model is also exported in ONNX format at {}'. format(os.path.realpath(args.onnx_export))) model.eval() dummy_input = torch.LongTensor(seq_len * batch_size).zero_().view(-1, batch_size).to(device) hidden = model.init_hidden(batch_size) torch.onnx.export(model, (dummy_input, hidden), path)
def evaluate(data_source, batch_size=10, window=args.window): # Turn on evaluation mode which disables dropout. if args.model == 'QRNN': model.reset() model.eval() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) next_word_history = None pointer_history = None for i in range(0, data_source.size(0) - 1, args.bptt): if i > 0: print(i, len(data_source), math.exp(total_loss / i)) data, targets = get_batch(data_source, i, evaluation=True, args=args) output, hidden, rnn_outs, _ = model(data, hidden, return_h=True) rnn_out = rnn_outs[-1].squeeze() output_flat = output.view(-1, ntokens) ### # Fill pointer history start_idx = len(next_word_history) if next_word_history is not None else 0 next_word_history = torch.cat([one_hot(t.data[0], ntokens) for t in targets]) if next_word_history is None else torch.cat([next_word_history, torch.cat([one_hot(t.data[0], ntokens) for t in targets])]) #print(next_word_history) pointer_history = Variable(rnn_out.data) if pointer_history is None else torch.cat([pointer_history, Variable(rnn_out.data)], dim=0) #print(pointer_history) ### # Built-in cross entropy # total_loss += len(data) * criterion(output_flat, targets).data[0] ### # Manual cross entropy # softmax_output_flat = torch.nn.functional.softmax(output_flat) # soft = torch.gather(softmax_output_flat, dim=1, index=targets.view(-1, 1)) # entropy = -torch.log(soft) # total_loss += len(data) * entropy.mean().data[0] ### # Pointer manual cross entropy loss = 0 softmax_output_flat = torch.nn.functional.softmax(output_flat) for idx, vocab_loss in enumerate(softmax_output_flat): p = vocab_loss if start_idx + idx > window: valid_next_word = next_word_history[start_idx + idx - window:start_idx + idx] valid_pointer_history = pointer_history[start_idx + idx - window:start_idx + idx] logits = torch.mv(valid_pointer_history, rnn_out[idx]) theta = args.theta ptr_attn = torch.nn.functional.softmax(theta * logits).view(-1, 1) ptr_dist = (ptr_attn.expand_as(valid_next_word) * valid_next_word).sum(0).squeeze() lambdah = args.lambdasm p = lambdah * ptr_dist + (1 - lambdah) * vocab_loss ### target_loss = p[targets[idx].data] loss += (-torch.log(target_loss)).data[0] total_loss += loss / batch_size ### hidden = repackage_hidden(hidden) next_word_history = next_word_history[-window:] pointer_history = pointer_history[-window:] return total_loss / len(data_source)
def valid(epoch, quesfeaShu, labelShu, lengthShu): losses = AverageMeter() top1 = AverageMeter() model.eval() start_time = time.time() for i in range(0, len(quesfeaShu) / args.batch_size): if i == len(quesfeaShu) / args.batch_size - 1: batchend = len(quesfeaShu) else: batchend = (i + 1) * (args.batch_size) # print batchend batchstart = i * (args.batch_size) batch_size = batchend - batchstart quesfeabatch = [] labelbatch = [] lengthbatch = [] quesfeaOri = quesfeaShu[batchstart:batchend] labelOri = labelShu[batchstart:batchend] lengthOri = lengthShu[batchstart:batchend] idxbatch = sorted(range(len(lengthOri)), key=lambda x: lengthOri[x], reverse=True) for j in range(len(idxbatch)): quesfeabatch.append(quesfeaOri[idxbatch[j]]) labelbatch.append(labelOri[idxbatch[j]]) lengthbatch.append(lengthOri[idxbatch[j]]) questrainarray = np.asarray(quesfeabatch) labeltrainarray = np.asarray(labelbatch) lengthtrainarray = np.asarray(lengthbatch) tmp = [questrainarray, labeltrainarray, lengthtrainarray] tmp = [Variable(torch.from_numpy(_), requires_grad=False) for _ in tmp] trques, trlabel, length = tmp if args.cuda: trlabel.cuda() output = model(trques, length) # print output loss = criterion(output, trlabel) / (batch_size) prec1, = accuracy(output.data, trlabel.data, topk=(1,), ori_label=labeltrainarray) # label 0 or 1 losses.update(loss.data[0], batch_size) top1.update(prec1[0], batch_size) # loss.backward() # `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs. torch.nn.utils.clip_grad_norm(model.parameters(), args.clip) print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'batch_valid ' + str(i) # update better performance model global best_score if top1.avg > best_score: torch.save(model, args.save) print 'save model' best_score = top1.avg print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'epoch_valid ' + str(epoch)
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() if args.model == 'QRNN': model.reset() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args, evaluation=True) output, hidden = model(data, hidden) total_loss += len(data) * criterion(model.decoder.weight, model.decoder.bias, output, targets).data hidden = repackage_hidden(hidden) return total_loss.item() / len(data_source)
def evaluate(data_source): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(eval_batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, evaluation=True) output, hidden = model(data, hidden) output_flat = output.view(-1, ntokens) total_loss += len(data) * criterion(output_flat, targets).data hidden = repackage_hidden(hidden) return total_loss[0] / len(data_source)
def valid(epoch, quesfeaShu, labelShu, lengthShu): top1 = AverageMeter() model.eval() for i in range(0, len(quesfeaShu) / args.batch_size): if i == len(quesfeaShu) / args.batch_size - 1: batchend = len(quesfeaShu) else: batchend = (i + 1) * (args.batch_size) # print batchend batchstart = i * (args.batch_size) batch_size = batchend - batchstart quesfeabatch = [] labelbatch = [] lengthbatch = [] quesfeaOri = quesfeaShu[batchstart:batchend] labelOri = labelShu[batchstart:batchend] lengthOri = lengthShu[batchstart:batchend] idxbatch = sorted(range(len(lengthOri)), key=lambda x: lengthOri[x], reverse=True) for j in range(len(idxbatch)): quesfeabatch.append(quesfeaOri[idxbatch[j]]) labelbatch.append(labelOri[idxbatch[j]]) lengthbatch.append(lengthOri[idxbatch[j]]) questrainarray = np.asarray(quesfeabatch) labeltrainarray = np.asarray(labelbatch) lengthtrainarray = np.asarray(lengthbatch) tmp = [questrainarray, labeltrainarray, lengthtrainarray] tmp = [Variable(torch.from_numpy(_), requires_grad=False) for _ in tmp] trques, trlabel, length = tmp if args.cuda: trlabel.cuda() output = model(trques, length) loss = criterion(output, trlabel) / (batch_size) prec1, = accuracy(output.data, trlabel.data, topk=(1,)) top1.update(prec1[0], batch_size) print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'batch_valid ' + str(i) global best_score if top1.avg > best_score: torch.save(model, args.save) print 'save model' best_score = top1.avg print str(top1.avg) + ' ' + str(loss.data[0]) + ' ' + 'epoch_valid ' + str(epoch)
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args, evaluation=True) targets = targets.view(-1) log_prob, hidden = parallel_model(data, hidden) loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)), targets).data total_loss += len(data) * loss hidden = repackage_hidden(hidden) return total_loss[0] / len(data_source)
def test_model(loader, model): """ Help function that tests the models's performance on a dataset :param: loader: data loader for the dataset to test against """ correct = 0 total = 0 model.eval() for data, labels in loader: if args.cuda: data, labels = data.cuda(), labels.cuda() data = Variable(data) outputs = model(data) predicted = (outputs.max(1)[1].data.long()).view(-1) total += labels.size(0) correct += (predicted == labels).sum() model.train() return 100 * correct / total
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) with torch.no_grad(): for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args) targets = targets.view(-1) log_prob, hidden = parallel_model(data, hidden) loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)), targets).data total_loss += len(data) * loss hidden = repackage_hidden(hidden) return total_loss.item() / len(data_source)
def evaluate(data_source): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0 hidden = model.init_hidden(eval_batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args.bptt, evaluation=True) #> output has size seq_length x batch_size x vocab_size output, hidden = model(data, None) #> output_flat has size num_targets x vocab_size (batches are stacked together) #> ! important, otherwise softmax computation (e.g. with F.softmax()) is incorrect output_flat = output.view(-1, ntokens) #output_candidates_info(output_flat.data, targets.data) total_loss += len(data) * nn.CrossEntropyLoss()(output_flat, targets).data # hidden = repackage_hidden(hidden) return total_loss[0] /len(data_source)
def evaluate(data_source): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0. ntokens = len(corpus.dictionary) if args.model != 'Transformer': hidden = model.init_hidden(eval_batch_size) with torch.no_grad(): for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i) if args.model == 'Transformer': output = model(data) else: output, hidden = model(data, hidden) hidden = repackage_hidden(hidden) output_flat = output.view(-1, ntokens) total_loss += len(data) * criterion(output_flat, targets).item() return total_loss / (len(data_source) - 1)
def evaluate(): running_loss_eval = 0.0 model.eval() for i, (input, mask, name) in enumerate(val_loader): input, mask = input[0].to(device=device)[None,:,:,:], mask[0].to(device=device)[None,:,:,:] with torch.no_grad(): output = model(input) # torch.where(outputs > 0.5, torch.ones(1).cuda(),torch.zeros(1).cuda()) loss = criterion(output, mask) loss_item = loss.item() running_loss_eval += loss_item print(f"Eval: {epoch}, iteration: {i} of {len(val_loader)}, loss: {loss_item}") eval_loss.append(running_loss_eval / len(val_loader))
def evaluate(data_source): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0 ntokens = len(corpus.dictionary) if (not args.single) and (torch.cuda.device_count() > 1): #"module" is necessary when using DataParallel hidden = model.module.init_hidden(eval_batch_size) else: hidden = model.init_hidden(eval_batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, evaluation=True) output, hidden = model(data, hidden) output_flat = output.view(-1, ntokens) curr_loss = len(data) * criterion(output_flat, targets).data total_loss += curr_loss hidden = repackage_hidden(hidden) return total_loss[0] / len(data_source)
def evaluate(source): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0. if args.model != 'Transformer': hidden = model.init_hidden(eval_batch_size) # Speed up evaluation with torch.no_grad() with torch.no_grad(): for i in range(0, source.size(0) - 1, args.seq_len): data, targets = get_batch(source, i) if args.model == 'Transformer': output = model(data) else: output, hidden = model(data, hidden) hidden = repackage_hidden(hidden) loss = criterion(output.view(-1, ntokens), targets) total_loss += len(data) * loss.item() return total_loss / (len(source) - 1)
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. if args.model == 'QRNN': model.reset() model.eval() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args, evaluation=True) output, hidden = model(data, hidden) if isinstance(criterion, SplitCrossEntropyLoss): total_loss += len(data) * criterion( model.decoder.weight, model.decoder.bias, output, targets).data else: output_flat = output.view(-1, ntokens) total_loss += len(data) * criterion(output_flat, targets).data hidden = repackage_hidden(hidden) return total_loss.item() / len(data_source)
def train_step(input_data, y, optimizer, criterion, dev_data, batch_size=config.batch_size): model.train() count = 0 total_loss = 0 for i in range(0, len(y), batch_size): optimizer.zero_grad() loss = 0 ''' word_input = input_data[0] pos_input = input_data[1] label_input = input_data[2] ''' input_batch, y_batch = get_batch(input_data, y, i, batch_size) print("run minibatch :%d " % i) #input_batch,y_batch=get_batch(input_data,y,i,batch_size) y_batch = torch.from_numpy(y_batch.nonzero()[1]).long().to(device) y_predict_logits = model(input_batch) loss = criterion(y_predict_logits, y_batch) loss.backward() optimizer.step() total_loss += loss.item() print("minibatch : %d ,loss: %.5f" % (i, loss.item())) count += 1 print("-----------------------------------------------") print("avg loss : %.5f" % (total_loss / count)) print("-----------------------------------------------") print("Evauating on dev set...") model.eval() UAS = dev_step(dev_data) print("dev UAS :%.3f" % UAS) return UAS
def F1_matrix_score_for_data(model, A, data_loader, batch_size, top_k): device = model.device nb_batch = len(data_loader.dataset) // batch_size if len(data_loader.dataset) % batch_size == 0: total_batch = nb_batch else: total_batch = nb_batch + 1 print("Total batch: ", total_batch) list_R_score = [] list_P_score = [] list_F1_score = [] model.eval() with torch.no_grad(): for i, data_pack in enumerate(data_loader, 0): data_x, data_seq_len, data_y = data_pack x_ = data_x.to_dense().to(dtype=model.dtype, device=device) real_batch_size = x_.size()[0] # hidden = model.init_hidden(real_batch_size) y_ = data_y.to(dtype=model.dtype, device=device) logits_ = model(A, data_seq_len, x_) predict_basket = utils.predict_top_k(logits_, top_k, real_batch_size, model.nb_items) target_basket_np = y_.cpu().numpy() correct_predict = predict_basket * target_basket_np nb_correct = np.count_nonzero(correct_predict, axis=1) actual_basket_size = np.count_nonzero(target_basket_np, axis=1) batch_recall = nb_correct / actual_basket_size batch_precision = nb_correct / top_k batch_f1 = np.zeros_like(nb_correct, dtype=float) for i in range(len(batch_f1)): if (nb_correct[i] > 0): batch_f1[i] = (2 * (batch_precision[i] * batch_recall[i])) / ( batch_precision[i] + batch_recall[i]) list_P_score.append(batch_precision[i]) list_R_score.append(batch_recall[i]) list_F1_score.append(batch_f1[i]) # print(list_MRR_score) # print("MRR score: %.6f" % np.array(list_MRR_score).mean()) return np.array(list_R_score).mean(), np.array( list_P_score).mean(), np.array(list_F1_score).mean()
def train_loop(model, loss_function, optimizer, scheduler, args): training_loss = [] train_ldr = train_dataloader(args) validation_loss = [] test_ldr = test_dataloader(args) for epoch in range(1, args.epochs + 1): ## Training train_loss = 0 for batch_idx, batch in enumerate(train_ldr): # Sets the model into training mode model.train() loss = training_step(batch, batch_idx, model, loss_function, optimizer, args.device, train_on_y2=args.train_on_y2) train_loss += loss.item() if batch_idx % args.log_interval == 0: x,y, y2=batch print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.2e}'.format(epoch, batch_idx * len(x), len(train_ldr.dataset),100. * batch_idx / len(train_ldr), loss.item() / len(x))) average_loss = train_loss / len(train_ldr.dataset) print('====> Epoch: {} Average loss: {:.2e}'.format(epoch, average_loss)) training_loss.append(average_loss) scheduler.step() ## Testing test_loss = 0 for batch_idx, batch in enumerate(test_ldr): model.eval() loss = validation_step(batch, batch_idx, model, loss_function, args.device, train_on_y2=args.train_on_y2) test_loss += loss.item() average_loss_val = test_loss / len(test_ldr.dataset) print('====> validation loss: {:.2e}'.format(average_loss_val)) validation_loss.append(average_loss_val) if epoch % 100 == 0: checkpoint_save(epoch, model, optimizer, training_loss, validation_loss, args.model_name, args.locations, args) # Save the final model torch.save({'epoch':epoch, 'model_state_dict':model.state_dict(), 'optimizer_state_dict':optimizer.state_dict(), 'training_loss':training_loss, 'validation_loss':validation_loss, 'arguments':args}, args.locations['model_loc']+'/'+args.model_name) return training_loss, validation_loss
def evaluate(data_source): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0. ntokens = len(corpus.dictionary) # 2*10*200 hidden = model.init_hidden(eval_batch_size) with torch.no_grad(): for i in range(0, data_source.size(0) - 1, args.bptt): # data:35*10, targets:350 data, targets = get_batch(data_source, i) # 参数:(35*10, 2*10*200), output=(35,10,33278) hidden=(2,10,200) output, hidden = model(data, hidden) # output_flat == > 350*33278 output_flat = output.view(-1, ntokens) # len(data)=35, 称为sequence length total_loss += len(data) * criterion(output_flat, targets).item() hidden = repackage_hidden(hidden) return total_loss / (len(data_source) - 1)
def test(model, perm=torch.arange(0, 224 * 224 * 3).long()): model.eval() test_loss = 0 correct = 0 for data, target in test_loader: output = model(data) test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss pred = output.data.max( 1, keepdim=True)[1] # get the index of the max log-probability correct += pred.eq(target.data.view_as(pred)).cpu().sum().item() test_loss /= len(test_loader.dataset) accuracy = 100. * correct / len(test_loader.dataset) accuracy_list.append(accuracy) print( '\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format( test_loss, correct, len(test_loader.dataset), accuracy))
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() if args.model == 'QRNN': model.reset() total_loss = 0 ntokens = len(corpus['words'].idx2word) all_outputs = [] for i in range(0, len(data_source['sentences']) - 1, batch_size): data, lengths, max_length, targets = get_batch(data_source, i, batch_size) cur_batch_size = data.size(1) hidden = model.init_hidden(cur_batch_size) output, hidden = model(data, lengths, max_length, hidden) for p in output.tolist(): all_outputs.append(p) loss = batch_size * criterion(output, targets.long()) total_loss += loss hidden = repackage_hidden(hidden) # return total_loss.item() / batch_size return total_loss.item() / len(data_source['sentences']), all_outputs
def evaluate(split): # Turn on evaluation mode which disables dropout. model.eval() total_loss, nbatches = 0, 0 ntokens = len(corpus.dictionary.idx2word) #2740 bsz = 64 hidden = model.init_hidden(bsz) for source, target in corpus.tokenize('./Data/Dev/dev.csv'): #print("Matrices") #print(hidden.shape) #print(source.shape) print(source.size()) print(hidden.size()) output, hidden = model(source, hidden) output_flat = output.view(-1, ntokens) total_loss += criterion(output_flat, target.view(-1)).data hidden = repackage_hidden(hidden) nbatches += 1 return total_loss[0] / nbatches
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() if args.model == 'QRNN': model.reset() loss_measure = AverageMeter() acc_measure = AverageMeter() ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args, evaluation=True) output, hidden = model(data, hidden) loss = criterion(model.decoder.weight, model.decoder.bias, output, targets).data loss_measure.update(float(loss), targets.nelement()) acc = float(accuracy(output.data, targets.data)[0]) acc_measure.update(acc, targets.nelement()) hidden = repackage_hidden(hidden) return loss_measure.avg, acc_measure.avg
def validate(val_loader, model, criterion): batch_time = AverageMeter() losses = AverageMeter() acc = AverageMeter() # switch to evaluate mode model.eval() end = time.time() for i, (images, labels) in enumerate(val_loader): image_var = torch.tensor(images).cuda(async=True) target = torch.tensor(labels).cuda(async=True) # 图片前传。验证和测试时不需要更新网络权重,所以使用torch.no_grad(),表示不计算梯度 with torch.no_grad(): y_pred = model(image_var) loss = criterion(y_pred, target) # measure accuracy and record loss prec, PRED_COUNT = accuracy(y_pred.data, labels, topk=(1, 1)) losses.update(loss.item(), images.size(0)) acc.update(prec, PRED_COUNT) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % print_freq == 0: print('TrainVal: [{0}/{1}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' 'Accuray {acc.val:.3f} ({acc.avg:.3f})'.format( i, len(val_loader), batch_time=batch_time, loss=losses, acc=acc)) print(' * Accuray {acc.avg:.3f}'.format(acc=acc), '(Previous Best Acc: %.3f)' % best_precision, ' * Loss {loss.avg:.3f}'.format(loss=losses), 'Previous Lowest Loss: %.3f)' % lowest_loss) return acc.avg, losses.avg
def check_accuracy(model: nn.Module, data_loader: DataLoader): """ if loader.dataset.train: print('Checking accuracy on validation set') else: print('Checking accuracy on test set') """ num_correct = 0 num_samples = 0 model.eval() # set model to evaluation mode with torch.no_grad(): for data, target, _ in data_loader: data, target = data.to(device=DEVICE), target.to(device=DEVICE) _, label, _ = model(data) num_correct += (label == target).cpu().int().item() num_samples += label.size(0) acc = float(num_correct) / num_samples print('Got {} / {} correct {:.2f} %'.format(num_correct, num_samples, 100 * acc)) return acc
def valid_model(): model.eval() running_loss = 0.0 running_corrects = 0 for img_inputs, labels, paths in valid_loader: img_inputs = img_inputs.to(device) labels = labels.to(device) tag_inputs = tag_generator(paths, img_tags, w2v).to(device) optimizer.zero_grad() with torch.set_grad_enabled(False): outputs = model(img_inputs, tag_inputs) loss = criterion(outputs, labels) _, preds = torch.max(outputs, 1) running_loss += loss.item() * img_inputs.size(0) running_corrects += torch.sum(preds == labels.data) epoch_loss = running_loss / len(valid_loader.dataset) epoch_acc = running_corrects.double() / len(valid_loader.dataset) print('valid Loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc)) return epoch_loss, epoch_acc
def test_model(): model.load_state_dict(torch.load(args.tag+'_backup/checkpoint.pt')) model.eval() running_loss = 0.0 running_corrects = 0 for img_inputs, labels, paths in test_loader: img_inputs = img_inputs.to(device) labels = labels.to(device) tag_inputs = tag_generator(paths, img_tags, w2v).to(device) optimizer.zero_grad() with torch.set_grad_enabled(False): outputs = model(img_inputs, tag_inputs) loss = criterion(outputs, labels) _, preds = torch.max(outputs, 1) running_loss += loss.item() * img_inputs.size(0) running_corrects += torch.sum(preds == labels.data) epoch_loss = running_loss / len(test_loader.dataset) epoch_acc = running_corrects.double() / len(test_loader.dataset) print('test Loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))
def evaluate(model, criterion, data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() model_now = model.module criterion_now = criterion.module if args.model == 'QRNN': model_now.reset() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model_now.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args, evaluation=True) output, hidden = model_now(data, hidden) criterion_now.replicate_weight_and_bias( torch.nn.Parameter(model.module.decoder.weight), torch.nn.Parameter(model.module.decoder.bias)) total_loss += len(data) * criterion_now(hiddens=output, targets=targets).data hidden = repackage_hidden(hidden) return total_loss.item() / len(data_source)
def evaluate(data_source): # Turn on evaluation mode which disables dropout. model.eval() eval_criterion = nn.CrossEntropyLoss() total_loss = 0. ntokens = len(corpus.dictionary) hidden = model.init_hidden(eval_batch_size) with torch.no_grad(): for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i) # gs534 add sentence resetting eosidx = corpus.dictionary.get_eos() output, hidden = model(data, hidden, eosidx) output_flat = output.view(-1, ntokens) total_loss += len(data) * eval_criterion(output_flat, targets).item() hidden = repackage_hidden(hidden) return total_loss / len(data_source)
def evaluate(data_source): # Turn on evaluation mode which disables dropout. with torch.no_grad(): model.eval() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden( eval_batch_size) #hidden size(nlayers, bsz, hdsize) for i in range(0, data_source.size(0) - 1, args.bptt): # iterate over every timestep data, targets = get_batch(data_source, i) output, hidden = model(data, hidden) # model input and output # inputdata size(bptt, bsz), and size(bptt, bsz, embsize) after embedding # output size(bptt*bsz, ntoken) total_loss += len(data) * criterion(output, targets).data hidden = repackage_hidden(hidden) return total_loss / len(data_source)
def MRR_score_for_data(model, A, data_loader, batch_size): device = model.device nb_batch = len(data_loader.dataset) // batch_size if len(data_loader.dataset) % batch_size == 0: total_batch = nb_batch else: total_batch = nb_batch + 1 print("Total batch: ", total_batch) list_MRR_score = [] model.eval() with torch.no_grad(): for i, data_pack in enumerate(data_loader, 0): data_x, data_seq_len, data_y = data_pack x_ = data_x.to_dense().to(dtype=model.dtype, device=device) real_batch_size = x_.size()[0] # hidden = model.init_hidden(real_batch_size) y_ = data_y.to(dtype=model.dtype, device=device) predict_ = model(A, data_seq_len, x_) sigmoid_pred = torch.sigmoid(predict_) sorted_rank, indices = torch.sort(sigmoid_pred, descending=True) for seq_idx, a_seq_idx in enumerate(y_): # print(seq_idx) idx_item_in_target_basket = (a_seq_idx == 1.0).nonzero( as_tuple=True)[0] # print(idx_item_in_target_basket) sum_of_rank_score = 0 for idx_item in idx_item_in_target_basket: item_rank = (indices[seq_idx] == idx_item).nonzero( as_tuple=True)[0].item() # print("Rank %d" % (item_rank + 1)) rank_score = 1 / (item_rank + 1) sum_of_rank_score += rank_score # print("sum of rank item in target: %.6f" % sum_of_rank_score) target_basket_size = idx_item_in_target_basket.size()[0] MRR_score = sum_of_rank_score / target_basket_size # print(MRR_score) list_MRR_score.append(MRR_score) # print(list_MRR_score) # print("MRR score: %.6f" % np.array(list_MRR_score).mean()) # print("MRR list len: %d" % len(list_MRR_score)) return np.array(list_MRR_score).mean()
def evaluate_model(model, val_loader, epoch, num_epochs, writer, current_lr, log_every=20): n_classes = model.n_classes metric = torch.nn.CrossEntropyLoss() model.eval() for m in model.modules(): if isinstance(m, nn.BatchNorm2d): m.train() m.weight.requires_grad = False m.bias.requires_grad = False y_probs = np.zeros((0, n_classes), np.float) losses, y_trues = [], [] for i, (image, label, case_id) in enumerate(val_loader): # if torch.cuda.is_available(): # image = image.cuda() # label = label.cuda() prediction = model.forward(image.float()) loss = metric(prediction, label.long()) loss_value = loss.item() losses.append(loss_value) y_prob = F.softmax(prediction, dim=1) y_probs = np.concatenate([y_probs, y_prob.detach().cpu().numpy()]) y_trues.append(label.item()) metric_collects = utils.calc_multi_cls_measures(y_probs, y_trues) n_iter = epoch * len(val_loader) + i writer.add_scalar('Val/Loss', loss_value, n_iter) if (i % log_every == 0) & (i > 0): prefix = '*Val|' utils.print_progress(epoch + 1, num_epochs, i, len(val_loader), np.mean(losses), current_lr, metric_collects, prefix=prefix) val_loss_epoch = np.round(np.mean(losses), 4) return val_loss_epoch, metric_collects
def evaluate(model, data_source, batch_size=10): model.eval() if args.model == 'QRNN': model.reset() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.seq_len): data, targets = get_batch(data_source, i, args, evaluation=True) output = model(data, hidden) if isinstance(output, tuple): output, hidden = output output_flat = output.view(-1, ntokens) total_loss += len(data) * criterion(output_flat, targets).data hidden = repackage_hidden(hidden) return total_loss[0] / len(data_source)
def evaluate(data_source, source_sampler, target_sampler, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() if args.model == 'QRNN': model.reset() total_loss = 0 hidden = model.init_hidden(batch_size) for source_sample, target_sample in zip(source_sampler, target_sampler): model.train() data = Variable(torch.stack([data_source[i] for i in source_sample]), volatile=True) targets = Variable(torch.stack([data_source[i] for i in target_sample])).view(-1) output, hidden = model(data, hidden) total_loss += len(data) * criterion( model.decoder.weight, model.decoder.bias, output, targets).data hidden = repackage_hidden(hidden) return total_loss[0] / len(data_source)
def evaluate(input1, input2, label): model.eval() output = model(input1) _, prediction = torch.max(output.data, 1) label = label.data count = 0 count0 = 0 count1 = 0 count2 = 0 for i in range(len(label)): if label[i] == prediction[i]: count += 1 return count / len(label)
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() if args.model == 'QRNN': model.reset() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): data, targets = get_batch(data_source, i, args, evaluation=True) output, hidden = model(data, hidden) input = torch.mm(model.decoder.weight, output.transpose( 0, 1)).transpose(0, 1) + model.decoder.bias if args.loss == 'splitcrossentropy': total_loss += len(data) * criterion( model.decoder.weight, model.decoder.bias, output, targets).data elif args.loss == 'focal': total_loss += len(data) * criterion(input, targets, test=True).data hidden = repackage_hidden(hidden) return total_loss.item() / len(data_source)
def evaluate(data_source, batch_size=10): # Turn on evaluation mode which disables dropout. model.eval() total_loss = 0 ntokens = len(corpus.dictionary) hidden = model.init_hidden(batch_size) for i in range(0, data_source.size(0) - 1, args.bptt): print(i, data_source.size(0) - 1) data, targets = get_batch(data_source, i, args, evaluation=True) targets = targets.view(-1) log_prob, hidden = parallel_model(data, hidden) loss = nn.functional.nll_loss(log_prob.view(-1, log_prob.size(2)), targets).data total_loss += loss * len(data) hidden = repackage_hidden(hidden) return total_loss[0] / len(data_source)
correct = pred.cpu().eq(indx_target).sum() acc = correct * 1.0 / len(data) print('Train Epoch: {} [{}/{}] Loss: {:.6f} Acc: {:.4f} lr: {:.2e}'.format( epoch, batch_idx * len(data), len(train_loader.dataset), loss.data[0], acc, optimizer.param_groups[0]['lr'])) elapse_time = time.time() - t_begin speed_epoch = elapse_time / (epoch + 1) speed_batch = speed_epoch / len(train_loader) eta = speed_epoch * args.epochs - elapse_time print("Elapsed {:.2f}s, {:.2f} s/epoch, {:.2f} s/batch, ets {:.2f}s".format( elapse_time, speed_epoch, speed_batch, eta)) misc.model_snapshot(model, os.path.join(args.logdir, 'latest.pth')) if epoch % args.test_interval == 0: model.eval() test_loss = 0 correct = 0 for data, target in test_loader: indx_target = target.clone() if args.cuda: data, target = data.cuda(), target.cuda() data, target = Variable(data, volatile=True), Variable(target) output = model(data) test_loss += F.cross_entropy(output, target).data[0] pred = output.data.max(1)[1] # get the index of the max log-probability correct += pred.cpu().eq(indx_target).sum() test_loss = test_loss / len(test_loader) # average over number of mini-batch acc = 100. * correct / len(test_loader.dataset) print('\tTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format(